# from tvm.script import ir as I # from tvm.script import tir as T # from tvm.script import relax as R @I.ir_module class Module: I.module_attrs({"system_lib_prefix": "gemma_q0f16_"}) @T.prim_func def gemma_q0f16_NT_matmul(var_rms_norm111: T.handle, model_layers_0_self_attn_qkv_proj_weight4: T.Buffer((2560, 2048), "float16"), var_NT_matmul: T.handle): T.func_attr({"tir.HoistIfThenElseExprWithBlock": 1, "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() rms_norm111 = T.match_buffer(var_rms_norm111, (batch_size, 1, 2048), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (batch_size, 1, 2560), "float16") # with T.block("root"): if T.tvm_thread_invariant(batch_size <= 2): with T.block("root"): T.reads() T.writes() NT_matmul_pad_local = T.alloc_buffer(((batch_size + 1) // 2 * 2, 1, 2560), "float16", scope="local") NT_matmul_pad_rf_local = T.alloc_buffer((256, (batch_size + 1) // 2 * 2, 1, 2560), "float16", scope="local") NT_matmul_pad_rf_local_1 = T.alloc_buffer((32, (batch_size + 1) // 2 * 2, 1, 2560), "float16", scope="local") for ax0_0 in T.thread_binding((batch_size + 1) // 2, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(640, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(2, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax0_1_init) v1 = T.axis.spatial(2560, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = T.float16(0.0) for ax2_fused_0 in T.serial(8, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(2, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax0_1) v1 = T.axis.spatial(2560, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1], rms_norm111[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], model_layers_0_self_attn_qkv_proj_weight4[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] + T.if_then_else(v0 < batch_size, rms_norm111[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0)) * model_layers_0_self_attn_qkv_proj_weight4[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused] for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(2): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(2560, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(2560, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1], NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1]) T.writes(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] + NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1] for ax2_fused_2, ax1 in T.grid(2, 2): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax1) v1 = T.axis.spatial(2560, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) T.writes(NT_matmul_pad_local[v0, 0, v1]) with T.init(): NT_matmul_pad_local[v0, 0, v1] = T.float16(0.0) NT_matmul_pad_local[v0, 0, v1] = NT_matmul_pad_local[v0, 0, v1] + NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] for ax0 in range(2): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_pad"): v0 = T.axis.spatial(batch_size, ax0_0 * 2 + ax0) v1 = T.axis.spatial(2560, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (batch_size + 1) // 2 < 0 or ax0_0 == 0) and ax0_0 * 2 + ax0 < batch_size) T.reads(NT_matmul_pad_local[v0, 0, v1]) T.writes(NT_matmul[v0, 0, v1]) NT_matmul[v0, 0, v1] = NT_matmul_pad_local[v0, 0, v1] else: if T.tvm_thread_invariant(batch_size <= 8): with T.block("root"): T.reads() T.writes() NT_matmul_pad_local = T.alloc_buffer(((batch_size + 3) // 4 * 4, 1, 2560), "float16", scope="local") NT_matmul_pad_rf_local = T.alloc_buffer((256, (batch_size + 3) // 4 * 4, 1, 2560), "float16", scope="local") NT_matmul_pad_rf_local_1 = T.alloc_buffer((32, (batch_size + 3) // 4 * 4, 1, 2560), "float16", scope="local") for ax0_0 in T.thread_binding((batch_size + 3) // 4, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(640, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(4, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax0_1_init) v1 = T.axis.spatial(2560, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = T.float16(0.0) for ax2_fused_0 in T.serial(8, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(4, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax0_1) v1 = T.axis.spatial(2560, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1], rms_norm111[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], model_layers_0_self_attn_qkv_proj_weight4[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] + T.if_then_else(v0 < batch_size, rms_norm111[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0)) * model_layers_0_self_attn_qkv_proj_weight4[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused] for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(4): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(2560, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(2560, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1], NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1]) T.writes(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] + NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1] for ax2_fused_2, ax1 in T.grid(2, 4): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax1) v1 = T.axis.spatial(2560, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) T.writes(NT_matmul_pad_local[v0, 0, v1]) with T.init(): NT_matmul_pad_local[v0, 0, v1] = T.float16(0.0) NT_matmul_pad_local[v0, 0, v1] = NT_matmul_pad_local[v0, 0, v1] + NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] for ax0 in range(4): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_pad"): v0 = T.axis.spatial(batch_size, ax0_0 * 4 + ax0) v1 = T.axis.spatial(2560, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (batch_size + 3) // 4 < 0 or ax0_0 == 0) and ax0_0 * 4 + ax0 < batch_size) T.reads(NT_matmul_pad_local[v0, 0, v1]) T.writes(NT_matmul[v0, 0, v1]) NT_matmul[v0, 0, v1] = NT_matmul_pad_local[v0, 0, v1] else: with T.block("root"): T.reads() T.writes() NT_matmul_reindex_pad_local = T.alloc_buffer((1, (batch_size + 7) // 8 * 8, 2560), "float16", scope="local") for ax0_ax1_0_fused in T.thread_binding((batch_size + 7) // 8, thread="blockIdx.y"): for ax2_0 in T.thread_binding(10, thread="blockIdx.x"): for ax1_1 in T.thread_binding(1, thread="vthread.y"): for ax2_1 in T.thread_binding(1, thread="vthread.x"): for ax1_2 in T.thread_binding(4, thread="threadIdx.y"): for ax2_2 in T.thread_binding(32, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 16, "pragma_unroll_explicit": 1}): for ax1_3_init, ax2_3_0_init in T.grid(2, 1): for ax2_3_1_init in T.vectorized(8): with T.block("NT_matmul_init"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3_init) v2 = T.axis.spatial(2560, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0_init * 8 + ax2_3_1_init) T.reads() T.writes(NT_matmul_reindex_pad_local[0, v1, v2]) NT_matmul_reindex_pad_local[0, v1, v2] = T.float16(0.0) for ax3_0, ax3_1, ax1_3, ax2_3_0 in T.grid(128, 16, 2, 1): for ax2_3_1 in T.vectorized(8): with T.block("NT_matmul_update"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3) v2 = T.axis.spatial(2560, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0 * 8 + ax2_3_1) v3 = T.axis.reduce(2048, ax3_0 * 16 + ax3_1) T.reads(NT_matmul_reindex_pad_local[0, v1, v2], rms_norm111[v1, 0, v3], model_layers_0_self_attn_qkv_proj_weight4[v2, v3]) T.writes(NT_matmul_reindex_pad_local[0, v1, v2]) NT_matmul_reindex_pad_local[0, v1, v2] = NT_matmul_reindex_pad_local[0, v1, v2] + T.if_then_else(v1 < batch_size, rms_norm111[v1, 0, v3], T.float16(0.0)) * model_layers_0_self_attn_qkv_proj_weight4[v2, v3] for ax0, ax1, ax2_0_1 in T.grid(1, 2, 1): for ax2_1_1 in T.vectorized(8): with T.block("NT_matmul_reindex_pad_local"): v0 = T.axis.spatial(1, ax0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1) v2 = T.axis.spatial(2560, ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1) T.where(ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1 < batch_size) T.reads(NT_matmul_reindex_pad_local[v0, v1, v2]) T.writes(NT_matmul[v1, 0, v2]) NT_matmul[v1, 0, v2] = NT_matmul_reindex_pad_local[v0, v1, v2] @T.prim_func def gemma_q0f16_NT_matmul1(var_reshape219: T.handle, model_layers_0_self_attn_o_proj_weight4: T.Buffer((2048, 2048), "float16"), var_NT_matmul: T.handle): T.func_attr({"tir.HoistIfThenElseExprWithBlock": 1, "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() reshape219 = T.match_buffer(var_reshape219, (batch_size, 1, 2048), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (batch_size, 1, 2048), "float16") # with T.block("root"): if T.tvm_thread_invariant(batch_size <= 2): with T.block("root"): T.reads() T.writes() NT_matmul_pad_local = T.alloc_buffer(((batch_size + 1) // 2 * 2, 1, 2048), "float16", scope="local") NT_matmul_pad_rf_local = T.alloc_buffer((256, (batch_size + 1) // 2 * 2, 1, 2048), "float16", scope="local") NT_matmul_pad_rf_local_1 = T.alloc_buffer((32, (batch_size + 1) // 2 * 2, 1, 2048), "float16", scope="local") for ax0_0 in T.thread_binding((batch_size + 1) // 2, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(512, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(2, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax0_1_init) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = T.float16(0.0) for ax2_fused_0 in T.serial(8, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(2, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax0_1) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1], reshape219[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], model_layers_0_self_attn_o_proj_weight4[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] + T.if_then_else(v0 < batch_size, reshape219[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0)) * model_layers_0_self_attn_o_proj_weight4[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused] for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(2): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1], NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1]) T.writes(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] + NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1] for ax2_fused_2, ax1 in T.grid(2, 2): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax1) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) T.writes(NT_matmul_pad_local[v0, 0, v1]) with T.init(): NT_matmul_pad_local[v0, 0, v1] = T.float16(0.0) NT_matmul_pad_local[v0, 0, v1] = NT_matmul_pad_local[v0, 0, v1] + NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] for ax0 in range(2): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_pad"): v0 = T.axis.spatial(batch_size, ax0_0 * 2 + ax0) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (batch_size + 1) // 2 < 0 or ax0_0 == 0) and ax0_0 * 2 + ax0 < batch_size) T.reads(NT_matmul_pad_local[v0, 0, v1]) T.writes(NT_matmul[v0, 0, v1]) NT_matmul[v0, 0, v1] = NT_matmul_pad_local[v0, 0, v1] else: if T.tvm_thread_invariant(batch_size <= 8): with T.block("root"): T.reads() T.writes() NT_matmul_pad_local = T.alloc_buffer(((batch_size + 3) // 4 * 4, 1, 2048), "float16", scope="local") NT_matmul_pad_rf_local = T.alloc_buffer((256, (batch_size + 3) // 4 * 4, 1, 2048), "float16", scope="local") NT_matmul_pad_rf_local_1 = T.alloc_buffer((32, (batch_size + 3) // 4 * 4, 1, 2048), "float16", scope="local") for ax0_0 in T.thread_binding((batch_size + 3) // 4, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(512, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(4, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax0_1_init) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = T.float16(0.0) for ax2_fused_0 in T.serial(8, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(4, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax0_1) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1], reshape219[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], model_layers_0_self_attn_o_proj_weight4[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] + T.if_then_else(v0 < batch_size, reshape219[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0)) * model_layers_0_self_attn_o_proj_weight4[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused] for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(4): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1], NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1]) T.writes(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] + NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1] for ax2_fused_2, ax1 in T.grid(2, 4): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax1) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) T.writes(NT_matmul_pad_local[v0, 0, v1]) with T.init(): NT_matmul_pad_local[v0, 0, v1] = T.float16(0.0) NT_matmul_pad_local[v0, 0, v1] = NT_matmul_pad_local[v0, 0, v1] + NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] for ax0 in range(4): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_pad"): v0 = T.axis.spatial(batch_size, ax0_0 * 4 + ax0) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (batch_size + 3) // 4 < 0 or ax0_0 == 0) and ax0_0 * 4 + ax0 < batch_size) T.reads(NT_matmul_pad_local[v0, 0, v1]) T.writes(NT_matmul[v0, 0, v1]) NT_matmul[v0, 0, v1] = NT_matmul_pad_local[v0, 0, v1] else: with T.block("root"): T.reads() T.writes() NT_matmul_reindex_pad_local = T.alloc_buffer((1, (batch_size + 7) // 8 * 8, 2048), "float16", scope="local") for ax0_ax1_0_fused in T.thread_binding((batch_size + 7) // 8, thread="blockIdx.y"): for ax2_0 in T.thread_binding(8, thread="blockIdx.x"): for ax1_1 in T.thread_binding(1, thread="vthread.y"): for ax2_1 in T.thread_binding(1, thread="vthread.x"): for ax1_2 in T.thread_binding(4, thread="threadIdx.y"): for ax2_2 in T.thread_binding(32, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 16, "pragma_unroll_explicit": 1}): for ax1_3_init, ax2_3_0_init in T.grid(2, 1): for ax2_3_1_init in T.vectorized(8): with T.block("NT_matmul_init"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3_init) v2 = T.axis.spatial(2048, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0_init * 8 + ax2_3_1_init) T.reads() T.writes(NT_matmul_reindex_pad_local[0, v1, v2]) NT_matmul_reindex_pad_local[0, v1, v2] = T.float16(0.0) for ax3_0, ax3_1, ax1_3, ax2_3_0 in T.grid(128, 16, 2, 1): for ax2_3_1 in T.vectorized(8): with T.block("NT_matmul_update"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3) v2 = T.axis.spatial(2048, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0 * 8 + ax2_3_1) v3 = T.axis.reduce(2048, ax3_0 * 16 + ax3_1) T.reads(NT_matmul_reindex_pad_local[0, v1, v2], reshape219[v1, 0, v3], model_layers_0_self_attn_o_proj_weight4[v2, v3]) T.writes(NT_matmul_reindex_pad_local[0, v1, v2]) NT_matmul_reindex_pad_local[0, v1, v2] = NT_matmul_reindex_pad_local[0, v1, v2] + T.if_then_else(v1 < batch_size, reshape219[v1, 0, v3], T.float16(0.0)) * model_layers_0_self_attn_o_proj_weight4[v2, v3] for ax0, ax1, ax2_0_1 in T.grid(1, 2, 1): for ax2_1_1 in T.vectorized(8): with T.block("NT_matmul_reindex_pad_local"): v0 = T.axis.spatial(1, ax0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1) v2 = T.axis.spatial(2048, ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1) T.where(ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1 < batch_size) T.reads(NT_matmul_reindex_pad_local[v0, v1, v2]) T.writes(NT_matmul[v1, 0, v2]) NT_matmul[v1, 0, v2] = NT_matmul_reindex_pad_local[v0, v1, v2] @T.prim_func def gemma_q0f16_NT_matmul10(rms_norm37: T.Buffer((1, 1, 2048), "float16"), model_layers_0_self_attn_qkv_proj_weight2: T.Buffer((2560, 2048), "float16"), NT_matmul: T.Buffer((1, 1, 2560), "float16")): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) # with T.block("root"): NT_matmul_rf_local = T.alloc_buffer((256, 1, 1, 2560), "float16", scope="local") NT_matmul_rf_local_1 = T.alloc_buffer((32, 1, 1, 2560), "float16", scope="local") model_layers_0_self_attn_qkv_proj_weight2_local = T.alloc_buffer((2560, 2048), "float16", scope="local") for u_fused_ax0_fused_fused_0 in T.thread_binding(1280, thread="blockIdx.x"): for u_fused_ax0_fused_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for u_fused_ax0_fused_fused_2_init in range(1): for ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused = T.axis.spatial(256, ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 * 8 + ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1_init) v0 = T.axis.spatial(2560, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2_init) T.reads() T.writes(NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused, 0, 0, v0]) NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused, 0, 0, v0] = T.float16(0.0) for ax1_fused_u_fused_0 in T.serial(8, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_ax1_fused_0 in range(8): for ax0_ax1_fused_1 in T.vectorized(1): with T.block("model_layers_0_self_attn_qkv_proj_weight2_local"): v0 = T.axis.spatial(2560, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1) v1 = T.axis.spatial(2048, ax1_fused_u_fused_0 * 256 + ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 * 8 + ax0_ax1_fused_0 + ax0_ax1_fused_1) T.reads(model_layers_0_self_attn_qkv_proj_weight2[v0, v1]) T.writes(model_layers_0_self_attn_qkv_proj_weight2_local[v0, v1]) model_layers_0_self_attn_qkv_proj_weight2_local[v0, v1] = model_layers_0_self_attn_qkv_proj_weight2[v0, v1] for u_fused_ax0_fused_fused_2, ax1_fused_u_fused_2 in T.grid(1, 1): for ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused = T.axis.spatial(256, ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 * 8 + ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1) v0 = T.axis.spatial(2560, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2) vax1_fused_u_fused_2, vax1_fused_u_fused_0 = T.axis.remap("RR", [ax1_fused_u_fused_2, ax1_fused_u_fused_0]) T.reads(NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused, 0, 0, v0], rms_norm37[0, 0, vax1_fused_u_fused_2 * 8 + vax1_fused_u_fused_0 * 256 + vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused], model_layers_0_self_attn_qkv_proj_weight2_local[v0, vax1_fused_u_fused_0 * 256 + vax1_fused_u_fused_2 * 8 + vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused]) T.writes(NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused, 0, 0, v0]) NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused, 0, 0, v0] = NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused, 0, 0, v0] + rms_norm37[0, 0, vax1_fused_u_fused_2 * 8 + vax1_fused_u_fused_0 * 256 + vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused] * model_layers_0_self_attn_qkv_proj_weight2_local[v0, vax1_fused_u_fused_0 * 256 + vax1_fused_u_fused_2 * 8 + vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused] for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax2_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2_fused_2_1 in T.vectorized(1): with T.block("NT_matmul_rf_init"): vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial(2560, u_fused_ax0_fused_fused_0 * 2 + ax2_fused_0_ax2_fused_1_fused + ax2_fused_2_0 + ax2_fused_2_1) T.reads() T.writes(NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0]) NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial(2560, u_fused_ax0_fused_fused_0 * 2 + ax2_fused_0_ax2_fused_1_fused + ax2_fused_2_0 + ax2_fused_2_1) T.reads(NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0], NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 * 8 + vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1, 0, 0, v0]) T.writes(NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0]) NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0] = NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0] + NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 * 8 + vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1, 0, 0, v0] for ax1_fused_2 in range(1): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial(2560, u_fused_ax0_fused_fused_0 * 2 + ax1_fused_0_ax1_fused_1_fused + ax1_fused_2) T.reads(NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0]) T.writes(NT_matmul[0, 0, v0]) with T.init(): NT_matmul[0, 0, v0] = T.float16(0.0) NT_matmul[0, 0, v0] = NT_matmul[0, 0, v0] + NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0] @T.prim_func def gemma_q0f16_NT_matmul11(reshape75: T.Buffer((1, 1, 2048), "float16"), model_layers_0_self_attn_o_proj_weight2: T.Buffer((2048, 2048), "float16"), NT_matmul: T.Buffer((1, 1, 2048), "float16")): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) # with T.block("root"): NT_matmul_rf_local = T.alloc_buffer((256, 1, 1, 2048), "float16", scope="local") NT_matmul_rf_local_1 = T.alloc_buffer((32, 1, 1, 2048), "float16", scope="local") model_layers_0_self_attn_o_proj_weight2_local = T.alloc_buffer((2048, 2048), "float16", scope="local") for u_fused_ax0_fused_fused_0 in T.thread_binding(1024, thread="blockIdx.x"): for u_fused_ax0_fused_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for u_fused_ax0_fused_fused_2_init in range(1): for ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused = T.axis.spatial(256, ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 * 8 + ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1_init) v0 = T.axis.spatial(2048, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2_init) T.reads() T.writes(NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused, 0, 0, v0]) NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused, 0, 0, v0] = T.float16(0.0) for ax1_fused_u_fused_0 in T.serial(8, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_ax1_fused_0 in range(8): for ax0_ax1_fused_1 in T.vectorized(1): with T.block("model_layers_0_self_attn_o_proj_weight2_local"): v0 = T.axis.spatial(2048, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1) v1 = T.axis.spatial(2048, ax1_fused_u_fused_0 * 256 + ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 * 8 + ax0_ax1_fused_0 + ax0_ax1_fused_1) T.reads(model_layers_0_self_attn_o_proj_weight2[v0, v1]) T.writes(model_layers_0_self_attn_o_proj_weight2_local[v0, v1]) model_layers_0_self_attn_o_proj_weight2_local[v0, v1] = model_layers_0_self_attn_o_proj_weight2[v0, v1] for u_fused_ax0_fused_fused_2, ax1_fused_u_fused_2 in T.grid(1, 1): for ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused = T.axis.spatial(256, ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 * 8 + ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1) v0 = T.axis.spatial(2048, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2) vax1_fused_u_fused_2, vax1_fused_u_fused_0 = T.axis.remap("RR", [ax1_fused_u_fused_2, ax1_fused_u_fused_0]) T.reads(NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused, 0, 0, v0], reshape75[0, 0, vax1_fused_u_fused_2 * 8 + vax1_fused_u_fused_0 * 256 + vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused], model_layers_0_self_attn_o_proj_weight2_local[v0, vax1_fused_u_fused_0 * 256 + vax1_fused_u_fused_2 * 8 + vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused]) T.writes(NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused, 0, 0, v0]) NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused, 0, 0, v0] = NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused, 0, 0, v0] + reshape75[0, 0, vax1_fused_u_fused_2 * 8 + vax1_fused_u_fused_0 * 256 + vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused] * model_layers_0_self_attn_o_proj_weight2_local[v0, vax1_fused_u_fused_0 * 256 + vax1_fused_u_fused_2 * 8 + vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused] for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax2_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2_fused_2_1 in T.vectorized(1): with T.block("NT_matmul_rf_init"): vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial(2048, u_fused_ax0_fused_fused_0 * 2 + ax2_fused_0_ax2_fused_1_fused + ax2_fused_2_0 + ax2_fused_2_1) T.reads() T.writes(NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0]) NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial(2048, u_fused_ax0_fused_fused_0 * 2 + ax2_fused_0_ax2_fused_1_fused + ax2_fused_2_0 + ax2_fused_2_1) T.reads(NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0], NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 * 8 + vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1, 0, 0, v0]) T.writes(NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0]) NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0] = NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0] + NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 * 8 + vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1, 0, 0, v0] for ax1_fused_2 in range(1): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial(2048, u_fused_ax0_fused_fused_0 * 2 + ax1_fused_0_ax1_fused_1_fused + ax1_fused_2) T.reads(NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0]) T.writes(NT_matmul[0, 0, v0]) with T.init(): NT_matmul[0, 0, v0] = T.float16(0.0) NT_matmul[0, 0, v0] = NT_matmul[0, 0, v0] + NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0] @T.prim_func def gemma_q0f16_NT_matmul12(rms_norm38: T.Buffer((1, 1, 2048), "float16"), model_layers_0_mlp_gate_up_proj_weight2: T.Buffer((32768, 2048), "float16"), NT_matmul: T.Buffer((1, 1, 32768), "float16")): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) # with T.block("root"): NT_matmul_rf_local = T.alloc_buffer((256, 1, 1, 32768), "float16", scope="local") NT_matmul_rf_local_1 = T.alloc_buffer((32, 1, 1, 32768), "float16", scope="local") model_layers_0_mlp_gate_up_proj_weight2_local = T.alloc_buffer((32768, 2048), "float16", scope="local") for u_fused_ax0_fused_fused_0 in T.thread_binding(16384, thread="blockIdx.x"): for u_fused_ax0_fused_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for u_fused_ax0_fused_fused_2_init in range(1): for ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused = T.axis.spatial(256, ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 * 8 + ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1_init) v0 = T.axis.spatial(32768, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2_init) T.reads() T.writes(NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused, 0, 0, v0]) NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused, 0, 0, v0] = T.float16(0.0) for ax1_fused_u_fused_0 in T.serial(8, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_ax1_fused_0 in range(8): for ax0_ax1_fused_1 in T.vectorized(1): with T.block("model_layers_0_mlp_gate_up_proj_weight2_local"): v0 = T.axis.spatial(32768, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1) v1 = T.axis.spatial(2048, ax1_fused_u_fused_0 * 256 + ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 * 8 + ax0_ax1_fused_0 + ax0_ax1_fused_1) T.reads(model_layers_0_mlp_gate_up_proj_weight2[v0, v1]) T.writes(model_layers_0_mlp_gate_up_proj_weight2_local[v0, v1]) model_layers_0_mlp_gate_up_proj_weight2_local[v0, v1] = model_layers_0_mlp_gate_up_proj_weight2[v0, v1] for u_fused_ax0_fused_fused_2, ax1_fused_u_fused_2 in T.grid(1, 1): for ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused = T.axis.spatial(256, ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 * 8 + ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1) v0 = T.axis.spatial(32768, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2) vax1_fused_u_fused_2, vax1_fused_u_fused_0 = T.axis.remap("RR", [ax1_fused_u_fused_2, ax1_fused_u_fused_0]) T.reads(NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused, 0, 0, v0], rms_norm38[0, 0, vax1_fused_u_fused_2 * 8 + vax1_fused_u_fused_0 * 256 + vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused], model_layers_0_mlp_gate_up_proj_weight2_local[v0, vax1_fused_u_fused_0 * 256 + vax1_fused_u_fused_2 * 8 + vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused]) T.writes(NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused, 0, 0, v0]) NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused, 0, 0, v0] = NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused, 0, 0, v0] + rms_norm38[0, 0, vax1_fused_u_fused_2 * 8 + vax1_fused_u_fused_0 * 256 + vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused] * model_layers_0_mlp_gate_up_proj_weight2_local[v0, vax1_fused_u_fused_0 * 256 + vax1_fused_u_fused_2 * 8 + vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused] for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax2_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2_fused_2_1 in T.vectorized(1): with T.block("NT_matmul_rf_init"): vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial(32768, u_fused_ax0_fused_fused_0 * 2 + ax2_fused_0_ax2_fused_1_fused + ax2_fused_2_0 + ax2_fused_2_1) T.reads() T.writes(NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0]) NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial(32768, u_fused_ax0_fused_fused_0 * 2 + ax2_fused_0_ax2_fused_1_fused + ax2_fused_2_0 + ax2_fused_2_1) T.reads(NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0], NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 * 8 + vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1, 0, 0, v0]) T.writes(NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0]) NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0] = NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0] + NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 * 8 + vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1, 0, 0, v0] for ax1_fused_2 in range(1): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial(32768, u_fused_ax0_fused_fused_0 * 2 + ax1_fused_0_ax1_fused_1_fused + ax1_fused_2) T.reads(NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0]) T.writes(NT_matmul[0, 0, v0]) with T.init(): NT_matmul[0, 0, v0] = T.float16(0.0) NT_matmul[0, 0, v0] = NT_matmul[0, 0, v0] + NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0] @T.prim_func def gemma_q0f16_NT_matmul13(mul20: T.Buffer((1, 1, 16384), "float16"), model_layers_0_mlp_down_proj_weight2: T.Buffer((2048, 16384), "float16"), NT_matmul: T.Buffer((1, 1, 2048), "float16")): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) # with T.block("root"): NT_matmul_rf_local = T.alloc_buffer((256, 1, 1, 2048), "float16", scope="local") NT_matmul_rf_local_1 = T.alloc_buffer((32, 1, 1, 2048), "float16", scope="local") model_layers_0_mlp_down_proj_weight2_local = T.alloc_buffer((2048, 16384), "float16", scope="local") for u_fused_ax0_fused_fused_0 in T.thread_binding(1024, thread="blockIdx.x"): for u_fused_ax0_fused_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for u_fused_ax0_fused_fused_2_init in range(1): for ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused = T.axis.spatial(256, ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 * 8 + ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1_init) v0 = T.axis.spatial(2048, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2_init) T.reads() T.writes(NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused, 0, 0, v0]) NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused, 0, 0, v0] = T.float16(0.0) for ax1_fused_u_fused_0 in T.serial(64, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_ax1_fused_0 in range(8): for ax0_ax1_fused_1 in T.vectorized(1): with T.block("model_layers_0_mlp_down_proj_weight2_local"): v0 = T.axis.spatial(2048, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1) v1 = T.axis.spatial(16384, ax1_fused_u_fused_0 * 256 + ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 * 8 + ax0_ax1_fused_0 + ax0_ax1_fused_1) T.reads(model_layers_0_mlp_down_proj_weight2[v0, v1]) T.writes(model_layers_0_mlp_down_proj_weight2_local[v0, v1]) model_layers_0_mlp_down_proj_weight2_local[v0, v1] = model_layers_0_mlp_down_proj_weight2[v0, v1] for u_fused_ax0_fused_fused_2, ax1_fused_u_fused_2 in T.grid(1, 1): for ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused = T.axis.spatial(256, ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 * 8 + ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1) v0 = T.axis.spatial(2048, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2) vax1_fused_u_fused_2, vax1_fused_u_fused_0 = T.axis.remap("RR", [ax1_fused_u_fused_2, ax1_fused_u_fused_0]) T.reads(NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused, 0, 0, v0], mul20[0, 0, vax1_fused_u_fused_2 * 8 + vax1_fused_u_fused_0 * 256 + vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused], model_layers_0_mlp_down_proj_weight2_local[v0, vax1_fused_u_fused_0 * 256 + vax1_fused_u_fused_2 * 8 + vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused]) T.writes(NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused, 0, 0, v0]) NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused, 0, 0, v0] = NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused, 0, 0, v0] + mul20[0, 0, vax1_fused_u_fused_2 * 8 + vax1_fused_u_fused_0 * 256 + vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused] * model_layers_0_mlp_down_proj_weight2_local[v0, vax1_fused_u_fused_0 * 256 + vax1_fused_u_fused_2 * 8 + vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused] for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax2_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2_fused_2_1 in T.vectorized(1): with T.block("NT_matmul_rf_init"): vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial(2048, u_fused_ax0_fused_fused_0 * 2 + ax2_fused_0_ax2_fused_1_fused + ax2_fused_2_0 + ax2_fused_2_1) T.reads() T.writes(NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0]) NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial(2048, u_fused_ax0_fused_fused_0 * 2 + ax2_fused_0_ax2_fused_1_fused + ax2_fused_2_0 + ax2_fused_2_1) T.reads(NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0], NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 * 8 + vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1, 0, 0, v0]) T.writes(NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0]) NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0] = NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0] + NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 * 8 + vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1, 0, 0, v0] for ax1_fused_2 in range(1): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial(2048, u_fused_ax0_fused_fused_0 * 2 + ax1_fused_0_ax1_fused_1_fused + ax1_fused_2) T.reads(NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0]) T.writes(NT_matmul[0, 0, v0]) with T.init(): NT_matmul[0, 0, v0] = T.float16(0.0) NT_matmul[0, 0, v0] = NT_matmul[0, 0, v0] + NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0] @T.prim_func def gemma_q0f16_NT_matmul14(rms_norm73: T.Buffer((1, 1, 2048), "float16"), var_model_embed_tokens_weight2: T.handle, var_NT_matmul: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) vocab_size = T.int32() model_embed_tokens_weight2 = T.match_buffer(var_model_embed_tokens_weight2, (vocab_size, 2048), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (1, 1, vocab_size)) # with T.block("root"): NT_matmul_rf_local = T.alloc_buffer((256, 1, 1, vocab_size), scope="local") NT_matmul_rf_local_1 = T.alloc_buffer((32, 1, 1, vocab_size), scope="local") model_embed_tokens_weight2_local = T.alloc_buffer((vocab_size, 2048), "float16", scope="local") for u_fused_ax0_fused_fused_0 in T.thread_binding((vocab_size + 1) // 2, thread="blockIdx.x"): for u_fused_ax0_fused_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for u_fused_ax0_fused_fused_2_init in range(1): for ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused = T.axis.spatial(256, ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 * 8 + ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1_init) v0 = T.axis.spatial(vocab_size, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2_init) T.where(u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2_init < vocab_size) T.reads() T.writes(NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused, 0, 0, v0]) NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused, 0, 0, v0] = T.float32(0.0) for ax1_fused_u_fused_0 in T.serial(8, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_ax1_fused_0 in range(8): for ax0_ax1_fused_1 in T.vectorized(1): with T.block("model_embed_tokens_weight2_local"): v0 = T.axis.spatial(vocab_size, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1) v1 = T.axis.spatial(2048, ax1_fused_u_fused_0 * 256 + ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 * 8 + ax0_ax1_fused_0 + ax0_ax1_fused_1) T.where(u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 < vocab_size) T.reads(model_embed_tokens_weight2[v0, v1]) T.writes(model_embed_tokens_weight2_local[v0, v1]) model_embed_tokens_weight2_local[v0, v1] = model_embed_tokens_weight2[v0, v1] for u_fused_ax0_fused_fused_2, ax1_fused_u_fused_2 in T.grid(1, 1): for ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused = T.axis.spatial(256, ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 * 8 + ax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1) v0 = T.axis.spatial(vocab_size, u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2) vax1_fused_u_fused_2, vax1_fused_u_fused_0 = T.axis.remap("RR", [ax1_fused_u_fused_2, ax1_fused_u_fused_0]) T.where(u_fused_ax0_fused_fused_0 * 2 + u_fused_ax0_fused_fused_1 + u_fused_ax0_fused_fused_2 < vocab_size) T.reads(NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused, 0, 0, v0], rms_norm73[0, 0, vax1_fused_u_fused_2 * 8 + vax1_fused_u_fused_0 * 256 + vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused], model_embed_tokens_weight2_local[v0, vax1_fused_u_fused_0 * 256 + vax1_fused_u_fused_2 * 8 + vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused]) T.writes(NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused, 0, 0, v0]) NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused, 0, 0, v0] = NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused, 0, 0, v0] + T.Cast("float32", rms_norm73[0, 0, vax1_fused_u_fused_2 * 8 + vax1_fused_u_fused_0 * 256 + vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused]) * T.Cast("float32", model_embed_tokens_weight2_local[v0, vax1_fused_u_fused_0 * 256 + vax1_fused_u_fused_2 * 8 + vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused]) for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax2_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2_fused_2_1 in T.vectorized(1): with T.block("NT_matmul_rf_init"): vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial(vocab_size, u_fused_ax0_fused_fused_0 * 2 + ax2_fused_0_ax2_fused_1_fused + ax2_fused_2_0 + ax2_fused_2_1) T.where(u_fused_ax0_fused_fused_0 * 2 + (ax2_fused_0_ax2_fused_1_fused % 2 + (ax2_fused_2_0 + ax2_fused_2_1)) < vocab_size) T.reads() T.writes(NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0]) NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0] = T.float32(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial(vocab_size, u_fused_ax0_fused_fused_0 * 2 + ax2_fused_0_ax2_fused_1_fused + ax2_fused_2_0 + ax2_fused_2_1) T.where(u_fused_ax0_fused_fused_0 * 2 + (ax2_fused_0_ax2_fused_1_fused % 2 + (ax2_fused_2_0 + ax2_fused_2_1)) < vocab_size) T.reads(NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0], NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 * 8 + vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1, 0, 0, v0]) T.writes(NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0]) NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0] = NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0] + NT_matmul_rf_local[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 * 8 + vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_1, 0, 0, v0] for ax1_fused_2 in range(1): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial(vocab_size, u_fused_ax0_fused_fused_0 * 2 + ax1_fused_0_ax1_fused_1_fused + ax1_fused_2) T.where(u_fused_ax0_fused_fused_0 * 2 + (ax1_fused_0_ax1_fused_1_fused % 2 + ax1_fused_2) < vocab_size) T.reads(NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0]) T.writes(NT_matmul[0, 0, v0]) with T.init(): NT_matmul[0, 0, v0] = T.float32(0.0) NT_matmul[0, 0, v0] = NT_matmul[0, 0, v0] + NT_matmul_rf_local_1[vax1_fused_u_fused_1_ax1_fused_u_fused_3_fused_0, 0, 0, v0] @T.prim_func def gemma_q0f16_NT_matmul2(var_rms_norm112: T.handle, model_layers_0_mlp_gate_up_proj_weight4: T.Buffer((32768, 2048), "float16"), var_NT_matmul: T.handle): T.func_attr({"tir.HoistIfThenElseExprWithBlock": 1, "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() rms_norm112 = T.match_buffer(var_rms_norm112, (batch_size, 1, 2048), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (batch_size, 1, 32768), "float16") # with T.block("root"): if T.tvm_thread_invariant(batch_size <= 2): with T.block("root"): T.reads() T.writes() NT_matmul_pad_local = T.alloc_buffer(((batch_size + 1) // 2 * 2, 1, 32768), "float16", scope="local") NT_matmul_pad_rf_local = T.alloc_buffer((256, (batch_size + 1) // 2 * 2, 1, 32768), "float16", scope="local") NT_matmul_pad_rf_local_1 = T.alloc_buffer((32, (batch_size + 1) // 2 * 2, 1, 32768), "float16", scope="local") for ax0_0 in T.thread_binding((batch_size + 1) // 2, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(8192, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(2, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax0_1_init) v1 = T.axis.spatial(32768, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = T.float16(0.0) for ax2_fused_0 in T.serial(8, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(2, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax0_1) v1 = T.axis.spatial(32768, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1], rms_norm112[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], model_layers_0_mlp_gate_up_proj_weight4[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] + T.if_then_else(v0 < batch_size, rms_norm112[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0)) * model_layers_0_mlp_gate_up_proj_weight4[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused] for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(2): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(32768, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(32768, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1], NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1]) T.writes(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] + NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1] for ax2_fused_2, ax1 in T.grid(2, 2): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax1) v1 = T.axis.spatial(32768, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) T.writes(NT_matmul_pad_local[v0, 0, v1]) with T.init(): NT_matmul_pad_local[v0, 0, v1] = T.float16(0.0) NT_matmul_pad_local[v0, 0, v1] = NT_matmul_pad_local[v0, 0, v1] + NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] for ax0 in range(2): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_pad"): v0 = T.axis.spatial(batch_size, ax0_0 * 2 + ax0) v1 = T.axis.spatial(32768, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (batch_size + 1) // 2 < 0 or ax0_0 == 0) and ax0_0 * 2 + ax0 < batch_size) T.reads(NT_matmul_pad_local[v0, 0, v1]) T.writes(NT_matmul[v0, 0, v1]) NT_matmul[v0, 0, v1] = NT_matmul_pad_local[v0, 0, v1] else: if T.tvm_thread_invariant(batch_size <= 8): with T.block("root"): T.reads() T.writes() NT_matmul_pad_local = T.alloc_buffer(((batch_size + 3) // 4 * 4, 1, 32768), "float16", scope="local") NT_matmul_pad_rf_local = T.alloc_buffer((256, (batch_size + 3) // 4 * 4, 1, 32768), "float16", scope="local") NT_matmul_pad_rf_local_1 = T.alloc_buffer((32, (batch_size + 3) // 4 * 4, 1, 32768), "float16", scope="local") for ax0_0 in T.thread_binding((batch_size + 3) // 4, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(8192, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(4, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax0_1_init) v1 = T.axis.spatial(32768, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = T.float16(0.0) for ax2_fused_0 in T.serial(8, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(4, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax0_1) v1 = T.axis.spatial(32768, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1], rms_norm112[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], model_layers_0_mlp_gate_up_proj_weight4[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] + T.if_then_else(v0 < batch_size, rms_norm112[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0)) * model_layers_0_mlp_gate_up_proj_weight4[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused] for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(4): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(32768, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(32768, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1], NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1]) T.writes(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] + NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1] for ax2_fused_2, ax1 in T.grid(2, 4): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax1) v1 = T.axis.spatial(32768, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) T.writes(NT_matmul_pad_local[v0, 0, v1]) with T.init(): NT_matmul_pad_local[v0, 0, v1] = T.float16(0.0) NT_matmul_pad_local[v0, 0, v1] = NT_matmul_pad_local[v0, 0, v1] + NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] for ax0 in range(4): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_pad"): v0 = T.axis.spatial(batch_size, ax0_0 * 4 + ax0) v1 = T.axis.spatial(32768, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (batch_size + 3) // 4 < 0 or ax0_0 == 0) and ax0_0 * 4 + ax0 < batch_size) T.reads(NT_matmul_pad_local[v0, 0, v1]) T.writes(NT_matmul[v0, 0, v1]) NT_matmul[v0, 0, v1] = NT_matmul_pad_local[v0, 0, v1] else: with T.block("root"): T.reads() T.writes() NT_matmul_reindex_pad_local = T.alloc_buffer((1, (batch_size + 7) // 8 * 8, 32768), "float16", scope="local") for ax0_ax1_0_fused in T.thread_binding((batch_size + 7) // 8, thread="blockIdx.y"): for ax2_0 in T.thread_binding(128, thread="blockIdx.x"): for ax1_1 in T.thread_binding(1, thread="vthread.y"): for ax2_1 in T.thread_binding(1, thread="vthread.x"): for ax1_2 in T.thread_binding(4, thread="threadIdx.y"): for ax2_2 in T.thread_binding(32, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 16, "pragma_unroll_explicit": 1}): for ax1_3_init, ax2_3_0_init in T.grid(2, 1): for ax2_3_1_init in T.vectorized(8): with T.block("NT_matmul_init"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3_init) v2 = T.axis.spatial(32768, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0_init * 8 + ax2_3_1_init) T.reads() T.writes(NT_matmul_reindex_pad_local[0, v1, v2]) NT_matmul_reindex_pad_local[0, v1, v2] = T.float16(0.0) for ax3_0, ax3_1, ax1_3, ax2_3_0 in T.grid(128, 16, 2, 1): for ax2_3_1 in T.vectorized(8): with T.block("NT_matmul_update"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3) v2 = T.axis.spatial(32768, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0 * 8 + ax2_3_1) v3 = T.axis.reduce(2048, ax3_0 * 16 + ax3_1) T.reads(NT_matmul_reindex_pad_local[0, v1, v2], rms_norm112[v1, 0, v3], model_layers_0_mlp_gate_up_proj_weight4[v2, v3]) T.writes(NT_matmul_reindex_pad_local[0, v1, v2]) NT_matmul_reindex_pad_local[0, v1, v2] = NT_matmul_reindex_pad_local[0, v1, v2] + T.if_then_else(v1 < batch_size, rms_norm112[v1, 0, v3], T.float16(0.0)) * model_layers_0_mlp_gate_up_proj_weight4[v2, v3] for ax0, ax1, ax2_0_1 in T.grid(1, 2, 1): for ax2_1_1 in T.vectorized(8): with T.block("NT_matmul_reindex_pad_local"): v0 = T.axis.spatial(1, ax0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1) v2 = T.axis.spatial(32768, ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1) T.where(ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1 < batch_size) T.reads(NT_matmul_reindex_pad_local[v0, v1, v2]) T.writes(NT_matmul[v1, 0, v2]) NT_matmul[v1, 0, v2] = NT_matmul_reindex_pad_local[v0, v1, v2] @T.prim_func def gemma_q0f16_NT_matmul3(var_mul58: T.handle, model_layers_0_mlp_down_proj_weight4: T.Buffer((2048, 16384), "float16"), var_NT_matmul: T.handle): T.func_attr({"tir.HoistIfThenElseExprWithBlock": 1, "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() mul58 = T.match_buffer(var_mul58, (batch_size, 1, 16384), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (batch_size, 1, 2048), "float16") # with T.block("root"): if T.tvm_thread_invariant(batch_size <= 2): with T.block("root"): T.reads() T.writes() NT_matmul_pad_local = T.alloc_buffer(((batch_size + 1) // 2 * 2, 1, 2048), "float16", scope="local") NT_matmul_pad_rf_local = T.alloc_buffer((256, (batch_size + 1) // 2 * 2, 1, 2048), "float16", scope="local") NT_matmul_pad_rf_local_1 = T.alloc_buffer((32, (batch_size + 1) // 2 * 2, 1, 2048), "float16", scope="local") for ax0_0 in T.thread_binding((batch_size + 1) // 2, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(512, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(2, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax0_1_init) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = T.float16(0.0) for ax2_fused_0 in T.serial(64, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(2, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax0_1) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1], mul58[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], model_layers_0_mlp_down_proj_weight4[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] + T.if_then_else(v0 < batch_size, mul58[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0)) * model_layers_0_mlp_down_proj_weight4[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused] for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(2): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1], NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1]) T.writes(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] + NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1] for ax2_fused_2, ax1 in T.grid(2, 2): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((batch_size + 1) // 2 * 2, ax0_0 * 2 + ax1) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) T.writes(NT_matmul_pad_local[v0, 0, v1]) with T.init(): NT_matmul_pad_local[v0, 0, v1] = T.float16(0.0) NT_matmul_pad_local[v0, 0, v1] = NT_matmul_pad_local[v0, 0, v1] + NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] for ax0 in range(2): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_pad"): v0 = T.axis.spatial(batch_size, ax0_0 * 2 + ax0) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (batch_size + 1) // 2 < 0 or ax0_0 == 0) and ax0_0 * 2 + ax0 < batch_size) T.reads(NT_matmul_pad_local[v0, 0, v1]) T.writes(NT_matmul[v0, 0, v1]) NT_matmul[v0, 0, v1] = NT_matmul_pad_local[v0, 0, v1] else: if T.tvm_thread_invariant(batch_size <= 8): with T.block("root"): T.reads() T.writes() NT_matmul_pad_local = T.alloc_buffer(((batch_size + 3) // 4 * 4, 1, 2048), "float16", scope="local") NT_matmul_pad_rf_local = T.alloc_buffer((256, (batch_size + 3) // 4 * 4, 1, 2048), "float16", scope="local") NT_matmul_pad_rf_local_1 = T.alloc_buffer((32, (batch_size + 3) // 4 * 4, 1, 2048), "float16", scope="local") for ax0_0 in T.thread_binding((batch_size + 3) // 4, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(512, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(4, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax0_1_init) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = T.float16(0.0) for ax2_fused_0 in T.serial(64, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(4, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax0_1) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1], mul58[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], model_layers_0_mlp_down_proj_weight4[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1]) NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] = NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, v0, 0, v1] + T.if_then_else(v0 < batch_size, mul58[v0, 0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0)) * model_layers_0_mlp_down_proj_weight4[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused] for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(4): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1], NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1]) T.writes(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] = NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] + NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, v0, 0, v1] for ax2_fused_2, ax1 in T.grid(2, 4): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((batch_size + 3) // 4 * 4, ax0_0 * 4 + ax1) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1]) T.writes(NT_matmul_pad_local[v0, 0, v1]) with T.init(): NT_matmul_pad_local[v0, 0, v1] = T.float16(0.0) NT_matmul_pad_local[v0, 0, v1] = NT_matmul_pad_local[v0, 0, v1] + NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, v0, 0, v1] for ax0 in range(4): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_pad"): v0 = T.axis.spatial(batch_size, ax0_0 * 4 + ax0) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (batch_size + 3) // 4 < 0 or ax0_0 == 0) and ax0_0 * 4 + ax0 < batch_size) T.reads(NT_matmul_pad_local[v0, 0, v1]) T.writes(NT_matmul[v0, 0, v1]) NT_matmul[v0, 0, v1] = NT_matmul_pad_local[v0, 0, v1] else: with T.block("root"): T.reads() T.writes() NT_matmul_reindex_pad_local = T.alloc_buffer((1, (batch_size + 7) // 8 * 8, 2048), "float16", scope="local") for ax0_ax1_0_fused in T.thread_binding((batch_size + 7) // 8, thread="blockIdx.y"): for ax2_0 in T.thread_binding(8, thread="blockIdx.x"): for ax1_1 in T.thread_binding(1, thread="vthread.y"): for ax2_1 in T.thread_binding(1, thread="vthread.x"): for ax1_2 in T.thread_binding(4, thread="threadIdx.y"): for ax2_2 in T.thread_binding(32, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 16, "pragma_unroll_explicit": 1}): for ax1_3_init, ax2_3_0_init in T.grid(2, 1): for ax2_3_1_init in T.vectorized(8): with T.block("NT_matmul_init"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3_init) v2 = T.axis.spatial(2048, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0_init * 8 + ax2_3_1_init) T.reads() T.writes(NT_matmul_reindex_pad_local[0, v1, v2]) NT_matmul_reindex_pad_local[0, v1, v2] = T.float16(0.0) for ax3_0, ax3_1, ax1_3, ax2_3_0 in T.grid(1024, 16, 2, 1): for ax2_3_1 in T.vectorized(8): with T.block("NT_matmul_update"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3) v2 = T.axis.spatial(2048, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0 * 8 + ax2_3_1) v3 = T.axis.reduce(16384, ax3_0 * 16 + ax3_1) T.reads(NT_matmul_reindex_pad_local[0, v1, v2], mul58[v1, 0, v3], model_layers_0_mlp_down_proj_weight4[v2, v3]) T.writes(NT_matmul_reindex_pad_local[0, v1, v2]) NT_matmul_reindex_pad_local[0, v1, v2] = NT_matmul_reindex_pad_local[0, v1, v2] + T.if_then_else(v1 < batch_size, mul58[v1, 0, v3], T.float16(0.0)) * model_layers_0_mlp_down_proj_weight4[v2, v3] for ax0, ax1, ax2_0_1 in T.grid(1, 2, 1): for ax2_1_1 in T.vectorized(8): with T.block("NT_matmul_reindex_pad_local"): v0 = T.axis.spatial(1, ax0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1) v2 = T.axis.spatial(2048, ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1) T.where(ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1 < batch_size) T.reads(NT_matmul_reindex_pad_local[v0, v1, v2]) T.writes(NT_matmul[v1, 0, v2]) NT_matmul[v1, 0, v2] = NT_matmul_reindex_pad_local[v0, v1, v2] @T.prim_func def gemma_q0f16_NT_matmul4(var_rms_norm147: T.handle, var_model_embed_tokens_weight4: T.handle, var_NT_matmul: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() rms_norm147 = T.match_buffer(var_rms_norm147, (batch_size, 1, 2048), "float16") vocab_size = T.int32() model_embed_tokens_weight4 = T.match_buffer(var_model_embed_tokens_weight4, (vocab_size, 2048), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (batch_size, 1, vocab_size)) # with T.block("root"): NT_matmul_reindex_pad_local = T.alloc_buffer((1, (batch_size + 7) // 8 * 8, (vocab_size + 255) // 256 * 256), scope="local") for ax0_ax1_0_fused in T.thread_binding((batch_size + 7) // 8, thread="blockIdx.y"): for ax2_0 in T.thread_binding((vocab_size + 255) // 256, thread="blockIdx.x"): for ax1_1 in T.thread_binding(1, thread="vthread.y"): for ax2_1 in T.thread_binding(1, thread="vthread.x"): for ax1_2 in T.thread_binding(4, thread="threadIdx.y"): for ax2_2 in T.thread_binding(32, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 16, "pragma_unroll_explicit": 1}): for ax1_3_init, ax2_3_0_init in T.grid(2, 1): for ax2_3_1_init in T.vectorized(8): with T.block("NT_matmul_init"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3_init) v2 = T.axis.spatial((vocab_size + 255) // 256 * 256, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0_init * 8 + ax2_3_1_init) T.reads() T.writes(NT_matmul_reindex_pad_local[0, v1, v2]) NT_matmul_reindex_pad_local[0, v1, v2] = T.float32(0.0) for ax3_0, ax3_1, ax1_3, ax2_3_0 in T.grid(128, 16, 2, 1): for ax2_3_1 in T.vectorized(8): with T.block("NT_matmul_update"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3) v2 = T.axis.spatial((vocab_size + 255) // 256 * 256, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0 * 8 + ax2_3_1) v3 = T.axis.reduce(2048, ax3_0 * 16 + ax3_1) T.reads(NT_matmul_reindex_pad_local[0, v1, v2], rms_norm147[v1, 0, v3], model_embed_tokens_weight4[v2, v3]) T.writes(NT_matmul_reindex_pad_local[0, v1, v2]) NT_matmul_reindex_pad_local[0, v1, v2] = NT_matmul_reindex_pad_local[0, v1, v2] + T.Cast("float32", T.if_then_else(v1 < batch_size, rms_norm147[v1, 0, v3], T.float16(0.0))) * T.Cast("float32", T.if_then_else(v2 < vocab_size, model_embed_tokens_weight4[v2, v3], T.float16(0.0))) for ax0, ax1, ax2_0_1 in T.grid(1, 2, 1): for ax2_1_1 in T.vectorized(8): with T.block("NT_matmul_reindex_pad_local"): v0 = T.axis.spatial(1, ax0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1) v2 = T.axis.spatial((vocab_size + 255) // 256 * 256, ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1) T.where(ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1 < batch_size and ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1 < vocab_size) T.reads(NT_matmul_reindex_pad_local[v0, v1, v2]) T.writes(NT_matmul[v1, 0, v2]) NT_matmul[v1, 0, v2] = NT_matmul_reindex_pad_local[v0, v1, v2] @T.prim_func def gemma_q0f16_NT_matmul5(var_rms_norm74: T.handle, model_layers_0_self_attn_qkv_proj_weight3: T.Buffer((2560, 2048), "float16"), var_NT_matmul: T.handle): T.func_attr({"tir.HoistIfThenElseExprWithBlock": 1, "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() rms_norm74 = T.match_buffer(var_rms_norm74, (1, seq_len, 2048), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (1, seq_len, 2560), "float16") # with T.block("root"): if T.tvm_thread_invariant(seq_len <= 2): with T.block("root"): T.reads() T.writes() NT_matmul_pad_local = T.alloc_buffer((1, (seq_len + 1) // 2 * 2, 2560), "float16", scope="local") NT_matmul_pad_rf_local = T.alloc_buffer((256, 1, (seq_len + 1) // 2 * 2, 2560), "float16", scope="local") NT_matmul_pad_rf_local_1 = T.alloc_buffer((32, 1, (seq_len + 1) // 2 * 2, 2560), "float16", scope="local") for ax0_0 in T.thread_binding((seq_len + 1) // 2, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(640, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(2, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax0_1_init) v1 = T.axis.spatial(2560, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = T.float16(0.0) for ax2_fused_0 in T.serial(8, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(2, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax0_1) v1 = T.axis.spatial(2560, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1], rms_norm74[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], model_layers_0_self_attn_qkv_proj_weight3[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] + T.if_then_else(v0 < seq_len, rms_norm74[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0)) * model_layers_0_self_attn_qkv_proj_weight3[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused] for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(2): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(2560, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(2560, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1], NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1]) T.writes(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] + NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1] for ax2_fused_2, ax1 in T.grid(2, 2): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax1) v1 = T.axis.spatial(2560, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) T.writes(NT_matmul_pad_local[0, v0, v1]) with T.init(): NT_matmul_pad_local[0, v0, v1] = T.float16(0.0) NT_matmul_pad_local[0, v0, v1] = NT_matmul_pad_local[0, v0, v1] + NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] for ax0 in range(2): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_pad"): v0 = T.axis.spatial(seq_len, ax0_0 * 2 + ax0) v1 = T.axis.spatial(2560, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (seq_len + 1) // 2 < 0 or ax0_0 == 0) and ax0_0 * 2 + ax0 < seq_len) T.reads(NT_matmul_pad_local[0, v0, v1]) T.writes(NT_matmul[0, v0, v1]) NT_matmul[0, v0, v1] = NT_matmul_pad_local[0, v0, v1] else: if T.tvm_thread_invariant(seq_len <= 8): with T.block("root"): T.reads() T.writes() NT_matmul_pad_local = T.alloc_buffer((1, (seq_len + 3) // 4 * 4, 2560), "float16", scope="local") NT_matmul_pad_rf_local = T.alloc_buffer((256, 1, (seq_len + 3) // 4 * 4, 2560), "float16", scope="local") NT_matmul_pad_rf_local_1 = T.alloc_buffer((32, 1, (seq_len + 3) // 4 * 4, 2560), "float16", scope="local") for ax0_0 in T.thread_binding((seq_len + 3) // 4, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(640, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(4, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax0_1_init) v1 = T.axis.spatial(2560, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = T.float16(0.0) for ax2_fused_0 in T.serial(8, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(4, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax0_1) v1 = T.axis.spatial(2560, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1], rms_norm74[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], model_layers_0_self_attn_qkv_proj_weight3[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] + T.if_then_else(v0 < seq_len, rms_norm74[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0)) * model_layers_0_self_attn_qkv_proj_weight3[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused] for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(4): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(2560, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(2560, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1], NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1]) T.writes(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] + NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1] for ax2_fused_2, ax1 in T.grid(2, 4): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax1) v1 = T.axis.spatial(2560, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) T.writes(NT_matmul_pad_local[0, v0, v1]) with T.init(): NT_matmul_pad_local[0, v0, v1] = T.float16(0.0) NT_matmul_pad_local[0, v0, v1] = NT_matmul_pad_local[0, v0, v1] + NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] for ax0 in range(4): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_pad"): v0 = T.axis.spatial(seq_len, ax0_0 * 4 + ax0) v1 = T.axis.spatial(2560, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (seq_len + 3) // 4 < 0 or ax0_0 == 0) and ax0_0 * 4 + ax0 < seq_len) T.reads(NT_matmul_pad_local[0, v0, v1]) T.writes(NT_matmul[0, v0, v1]) NT_matmul[0, v0, v1] = NT_matmul_pad_local[0, v0, v1] else: with T.block("root"): T.reads() T.writes() NT_matmul_reindex_pad_local = T.alloc_buffer((1, (seq_len + 7) // 8 * 8, 2560), "float16", scope="local") for ax0_ax1_0_fused in T.thread_binding((seq_len + 7) // 8, thread="blockIdx.y"): for ax2_0 in T.thread_binding(10, thread="blockIdx.x"): for ax1_1 in T.thread_binding(1, thread="vthread.y"): for ax2_1 in T.thread_binding(1, thread="vthread.x"): for ax1_2 in T.thread_binding(4, thread="threadIdx.y"): for ax2_2 in T.thread_binding(32, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 16, "pragma_unroll_explicit": 1}): for ax1_3_init, ax2_3_0_init in T.grid(2, 1): for ax2_3_1_init in T.vectorized(8): with T.block("NT_matmul_init"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3_init) v2 = T.axis.spatial(2560, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0_init * 8 + ax2_3_1_init) T.reads() T.writes(NT_matmul_reindex_pad_local[0, v1, v2]) NT_matmul_reindex_pad_local[0, v1, v2] = T.float16(0.0) for ax3_0, ax3_1, ax1_3, ax2_3_0 in T.grid(128, 16, 2, 1): for ax2_3_1 in T.vectorized(8): with T.block("NT_matmul_update"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3) v2 = T.axis.spatial(2560, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0 * 8 + ax2_3_1) v3 = T.axis.reduce(2048, ax3_0 * 16 + ax3_1) T.reads(NT_matmul_reindex_pad_local[0, v1, v2], rms_norm74[0, v1, v3], model_layers_0_self_attn_qkv_proj_weight3[v2, v3]) T.writes(NT_matmul_reindex_pad_local[0, v1, v2]) NT_matmul_reindex_pad_local[0, v1, v2] = NT_matmul_reindex_pad_local[0, v1, v2] + T.if_then_else(v1 < seq_len, rms_norm74[0, v1, v3], T.float16(0.0)) * model_layers_0_self_attn_qkv_proj_weight3[v2, v3] for ax0, ax1, ax2_0_1 in T.grid(1, 2, 1): for ax2_1_1 in T.vectorized(8): with T.block("NT_matmul_reindex_pad_local"): v0 = T.axis.spatial(1, ax0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1) v2 = T.axis.spatial(2560, ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1) T.where(ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1 < seq_len) T.reads(NT_matmul_reindex_pad_local[v0, v1, v2]) T.writes(NT_matmul[0, v1, v2]) NT_matmul[0, v1, v2] = NT_matmul_reindex_pad_local[v0, v1, v2] @T.prim_func def gemma_q0f16_NT_matmul6(var_reshape147: T.handle, model_layers_0_self_attn_o_proj_weight3: T.Buffer((2048, 2048), "float16"), var_NT_matmul: T.handle): T.func_attr({"tir.HoistIfThenElseExprWithBlock": 1, "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() reshape147 = T.match_buffer(var_reshape147, (1, seq_len, 2048), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (1, seq_len, 2048), "float16") # with T.block("root"): if T.tvm_thread_invariant(seq_len <= 2): with T.block("root"): T.reads() T.writes() NT_matmul_pad_local = T.alloc_buffer((1, (seq_len + 1) // 2 * 2, 2048), "float16", scope="local") NT_matmul_pad_rf_local = T.alloc_buffer((256, 1, (seq_len + 1) // 2 * 2, 2048), "float16", scope="local") NT_matmul_pad_rf_local_1 = T.alloc_buffer((32, 1, (seq_len + 1) // 2 * 2, 2048), "float16", scope="local") for ax0_0 in T.thread_binding((seq_len + 1) // 2, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(512, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(2, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax0_1_init) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = T.float16(0.0) for ax2_fused_0 in T.serial(8, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(2, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax0_1) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1], reshape147[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], model_layers_0_self_attn_o_proj_weight3[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] + T.if_then_else(v0 < seq_len, reshape147[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0)) * model_layers_0_self_attn_o_proj_weight3[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused] for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(2): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1], NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1]) T.writes(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] + NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1] for ax2_fused_2, ax1 in T.grid(2, 2): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax1) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) T.writes(NT_matmul_pad_local[0, v0, v1]) with T.init(): NT_matmul_pad_local[0, v0, v1] = T.float16(0.0) NT_matmul_pad_local[0, v0, v1] = NT_matmul_pad_local[0, v0, v1] + NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] for ax0 in range(2): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_pad"): v0 = T.axis.spatial(seq_len, ax0_0 * 2 + ax0) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (seq_len + 1) // 2 < 0 or ax0_0 == 0) and ax0_0 * 2 + ax0 < seq_len) T.reads(NT_matmul_pad_local[0, v0, v1]) T.writes(NT_matmul[0, v0, v1]) NT_matmul[0, v0, v1] = NT_matmul_pad_local[0, v0, v1] else: if T.tvm_thread_invariant(seq_len <= 8): with T.block("root"): T.reads() T.writes() NT_matmul_pad_local = T.alloc_buffer((1, (seq_len + 3) // 4 * 4, 2048), "float16", scope="local") NT_matmul_pad_rf_local = T.alloc_buffer((256, 1, (seq_len + 3) // 4 * 4, 2048), "float16", scope="local") NT_matmul_pad_rf_local_1 = T.alloc_buffer((32, 1, (seq_len + 3) // 4 * 4, 2048), "float16", scope="local") for ax0_0 in T.thread_binding((seq_len + 3) // 4, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(512, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(4, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax0_1_init) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = T.float16(0.0) for ax2_fused_0 in T.serial(8, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(4, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax0_1) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1], reshape147[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], model_layers_0_self_attn_o_proj_weight3[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] + T.if_then_else(v0 < seq_len, reshape147[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0)) * model_layers_0_self_attn_o_proj_weight3[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused] for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(4): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1], NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1]) T.writes(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] + NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1] for ax2_fused_2, ax1 in T.grid(2, 4): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax1) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) T.writes(NT_matmul_pad_local[0, v0, v1]) with T.init(): NT_matmul_pad_local[0, v0, v1] = T.float16(0.0) NT_matmul_pad_local[0, v0, v1] = NT_matmul_pad_local[0, v0, v1] + NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] for ax0 in range(4): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_pad"): v0 = T.axis.spatial(seq_len, ax0_0 * 4 + ax0) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (seq_len + 3) // 4 < 0 or ax0_0 == 0) and ax0_0 * 4 + ax0 < seq_len) T.reads(NT_matmul_pad_local[0, v0, v1]) T.writes(NT_matmul[0, v0, v1]) NT_matmul[0, v0, v1] = NT_matmul_pad_local[0, v0, v1] else: with T.block("root"): T.reads() T.writes() NT_matmul_reindex_pad_local = T.alloc_buffer((1, (seq_len + 7) // 8 * 8, 2048), "float16", scope="local") for ax0_ax1_0_fused in T.thread_binding((seq_len + 7) // 8, thread="blockIdx.y"): for ax2_0 in T.thread_binding(8, thread="blockIdx.x"): for ax1_1 in T.thread_binding(1, thread="vthread.y"): for ax2_1 in T.thread_binding(1, thread="vthread.x"): for ax1_2 in T.thread_binding(4, thread="threadIdx.y"): for ax2_2 in T.thread_binding(32, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 16, "pragma_unroll_explicit": 1}): for ax1_3_init, ax2_3_0_init in T.grid(2, 1): for ax2_3_1_init in T.vectorized(8): with T.block("NT_matmul_init"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3_init) v2 = T.axis.spatial(2048, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0_init * 8 + ax2_3_1_init) T.reads() T.writes(NT_matmul_reindex_pad_local[0, v1, v2]) NT_matmul_reindex_pad_local[0, v1, v2] = T.float16(0.0) for ax3_0, ax3_1, ax1_3, ax2_3_0 in T.grid(128, 16, 2, 1): for ax2_3_1 in T.vectorized(8): with T.block("NT_matmul_update"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3) v2 = T.axis.spatial(2048, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0 * 8 + ax2_3_1) v3 = T.axis.reduce(2048, ax3_0 * 16 + ax3_1) T.reads(NT_matmul_reindex_pad_local[0, v1, v2], reshape147[0, v1, v3], model_layers_0_self_attn_o_proj_weight3[v2, v3]) T.writes(NT_matmul_reindex_pad_local[0, v1, v2]) NT_matmul_reindex_pad_local[0, v1, v2] = NT_matmul_reindex_pad_local[0, v1, v2] + T.if_then_else(v1 < seq_len, reshape147[0, v1, v3], T.float16(0.0)) * model_layers_0_self_attn_o_proj_weight3[v2, v3] for ax0, ax1, ax2_0_1 in T.grid(1, 2, 1): for ax2_1_1 in T.vectorized(8): with T.block("NT_matmul_reindex_pad_local"): v0 = T.axis.spatial(1, ax0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1) v2 = T.axis.spatial(2048, ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1) T.where(ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1 < seq_len) T.reads(NT_matmul_reindex_pad_local[v0, v1, v2]) T.writes(NT_matmul[0, v1, v2]) NT_matmul[0, v1, v2] = NT_matmul_reindex_pad_local[v0, v1, v2] @T.prim_func def gemma_q0f16_NT_matmul7(var_rms_norm75: T.handle, model_layers_0_mlp_gate_up_proj_weight3: T.Buffer((32768, 2048), "float16"), var_NT_matmul: T.handle): T.func_attr({"tir.HoistIfThenElseExprWithBlock": 1, "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() rms_norm75 = T.match_buffer(var_rms_norm75, (1, seq_len, 2048), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (1, seq_len, 32768), "float16") # with T.block("root"): if T.tvm_thread_invariant(seq_len <= 2): with T.block("root"): T.reads() T.writes() NT_matmul_pad_local = T.alloc_buffer((1, (seq_len + 1) // 2 * 2, 32768), "float16", scope="local") NT_matmul_pad_rf_local = T.alloc_buffer((256, 1, (seq_len + 1) // 2 * 2, 32768), "float16", scope="local") NT_matmul_pad_rf_local_1 = T.alloc_buffer((32, 1, (seq_len + 1) // 2 * 2, 32768), "float16", scope="local") for ax0_0 in T.thread_binding((seq_len + 1) // 2, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(8192, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(2, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax0_1_init) v1 = T.axis.spatial(32768, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = T.float16(0.0) for ax2_fused_0 in T.serial(8, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(2, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax0_1) v1 = T.axis.spatial(32768, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1], rms_norm75[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], model_layers_0_mlp_gate_up_proj_weight3[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] + T.if_then_else(v0 < seq_len, rms_norm75[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0)) * model_layers_0_mlp_gate_up_proj_weight3[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused] for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(2): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(32768, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(32768, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1], NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1]) T.writes(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] + NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1] for ax2_fused_2, ax1 in T.grid(2, 2): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax1) v1 = T.axis.spatial(32768, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) T.writes(NT_matmul_pad_local[0, v0, v1]) with T.init(): NT_matmul_pad_local[0, v0, v1] = T.float16(0.0) NT_matmul_pad_local[0, v0, v1] = NT_matmul_pad_local[0, v0, v1] + NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] for ax0 in range(2): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_pad"): v0 = T.axis.spatial(seq_len, ax0_0 * 2 + ax0) v1 = T.axis.spatial(32768, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (seq_len + 1) // 2 < 0 or ax0_0 == 0) and ax0_0 * 2 + ax0 < seq_len) T.reads(NT_matmul_pad_local[0, v0, v1]) T.writes(NT_matmul[0, v0, v1]) NT_matmul[0, v0, v1] = NT_matmul_pad_local[0, v0, v1] else: if T.tvm_thread_invariant(seq_len <= 8): with T.block("root"): T.reads() T.writes() NT_matmul_pad_local = T.alloc_buffer((1, (seq_len + 3) // 4 * 4, 32768), "float16", scope="local") NT_matmul_pad_rf_local = T.alloc_buffer((256, 1, (seq_len + 3) // 4 * 4, 32768), "float16", scope="local") NT_matmul_pad_rf_local_1 = T.alloc_buffer((32, 1, (seq_len + 3) // 4 * 4, 32768), "float16", scope="local") for ax0_0 in T.thread_binding((seq_len + 3) // 4, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(8192, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(4, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax0_1_init) v1 = T.axis.spatial(32768, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = T.float16(0.0) for ax2_fused_0 in T.serial(8, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(4, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax0_1) v1 = T.axis.spatial(32768, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1], rms_norm75[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], model_layers_0_mlp_gate_up_proj_weight3[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] + T.if_then_else(v0 < seq_len, rms_norm75[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0)) * model_layers_0_mlp_gate_up_proj_weight3[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused] for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(4): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(32768, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(32768, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1], NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1]) T.writes(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] + NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1] for ax2_fused_2, ax1 in T.grid(2, 4): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax1) v1 = T.axis.spatial(32768, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) T.writes(NT_matmul_pad_local[0, v0, v1]) with T.init(): NT_matmul_pad_local[0, v0, v1] = T.float16(0.0) NT_matmul_pad_local[0, v0, v1] = NT_matmul_pad_local[0, v0, v1] + NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] for ax0 in range(4): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_pad"): v0 = T.axis.spatial(seq_len, ax0_0 * 4 + ax0) v1 = T.axis.spatial(32768, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (seq_len + 3) // 4 < 0 or ax0_0 == 0) and ax0_0 * 4 + ax0 < seq_len) T.reads(NT_matmul_pad_local[0, v0, v1]) T.writes(NT_matmul[0, v0, v1]) NT_matmul[0, v0, v1] = NT_matmul_pad_local[0, v0, v1] else: with T.block("root"): T.reads() T.writes() NT_matmul_reindex_pad_local = T.alloc_buffer((1, (seq_len + 7) // 8 * 8, 32768), "float16", scope="local") for ax0_ax1_0_fused in T.thread_binding((seq_len + 7) // 8, thread="blockIdx.y"): for ax2_0 in T.thread_binding(128, thread="blockIdx.x"): for ax1_1 in T.thread_binding(1, thread="vthread.y"): for ax2_1 in T.thread_binding(1, thread="vthread.x"): for ax1_2 in T.thread_binding(4, thread="threadIdx.y"): for ax2_2 in T.thread_binding(32, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 16, "pragma_unroll_explicit": 1}): for ax1_3_init, ax2_3_0_init in T.grid(2, 1): for ax2_3_1_init in T.vectorized(8): with T.block("NT_matmul_init"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3_init) v2 = T.axis.spatial(32768, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0_init * 8 + ax2_3_1_init) T.reads() T.writes(NT_matmul_reindex_pad_local[0, v1, v2]) NT_matmul_reindex_pad_local[0, v1, v2] = T.float16(0.0) for ax3_0, ax3_1, ax1_3, ax2_3_0 in T.grid(128, 16, 2, 1): for ax2_3_1 in T.vectorized(8): with T.block("NT_matmul_update"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3) v2 = T.axis.spatial(32768, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0 * 8 + ax2_3_1) v3 = T.axis.reduce(2048, ax3_0 * 16 + ax3_1) T.reads(NT_matmul_reindex_pad_local[0, v1, v2], rms_norm75[0, v1, v3], model_layers_0_mlp_gate_up_proj_weight3[v2, v3]) T.writes(NT_matmul_reindex_pad_local[0, v1, v2]) NT_matmul_reindex_pad_local[0, v1, v2] = NT_matmul_reindex_pad_local[0, v1, v2] + T.if_then_else(v1 < seq_len, rms_norm75[0, v1, v3], T.float16(0.0)) * model_layers_0_mlp_gate_up_proj_weight3[v2, v3] for ax0, ax1, ax2_0_1 in T.grid(1, 2, 1): for ax2_1_1 in T.vectorized(8): with T.block("NT_matmul_reindex_pad_local"): v0 = T.axis.spatial(1, ax0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1) v2 = T.axis.spatial(32768, ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1) T.where(ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1 < seq_len) T.reads(NT_matmul_reindex_pad_local[v0, v1, v2]) T.writes(NT_matmul[0, v1, v2]) NT_matmul[0, v1, v2] = NT_matmul_reindex_pad_local[v0, v1, v2] @T.prim_func def gemma_q0f16_NT_matmul8(var_mul39: T.handle, model_layers_0_mlp_down_proj_weight3: T.Buffer((2048, 16384), "float16"), var_NT_matmul: T.handle): T.func_attr({"tir.HoistIfThenElseExprWithBlock": 1, "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() mul39 = T.match_buffer(var_mul39, (1, seq_len, 16384), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (1, seq_len, 2048), "float16") # with T.block("root"): if T.tvm_thread_invariant(seq_len <= 2): with T.block("root"): T.reads() T.writes() NT_matmul_pad_local = T.alloc_buffer((1, (seq_len + 1) // 2 * 2, 2048), "float16", scope="local") NT_matmul_pad_rf_local = T.alloc_buffer((256, 1, (seq_len + 1) // 2 * 2, 2048), "float16", scope="local") NT_matmul_pad_rf_local_1 = T.alloc_buffer((32, 1, (seq_len + 1) // 2 * 2, 2048), "float16", scope="local") for ax0_0 in T.thread_binding((seq_len + 1) // 2, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(512, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(2, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax0_1_init) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = T.float16(0.0) for ax2_fused_0 in T.serial(64, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(2, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax0_1) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1], mul39[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], model_layers_0_mlp_down_proj_weight3[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] + T.if_then_else(v0 < seq_len, mul39[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0)) * model_layers_0_mlp_down_proj_weight3[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused] for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(2): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax2) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1], NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1]) T.writes(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] + NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1] for ax2_fused_2, ax1 in T.grid(2, 2): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((seq_len + 1) // 2 * 2, ax0_0 * 2 + ax1) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) T.writes(NT_matmul_pad_local[0, v0, v1]) with T.init(): NT_matmul_pad_local[0, v0, v1] = T.float16(0.0) NT_matmul_pad_local[0, v0, v1] = NT_matmul_pad_local[0, v0, v1] + NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] for ax0 in range(2): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_pad"): v0 = T.axis.spatial(seq_len, ax0_0 * 2 + ax0) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (seq_len + 1) // 2 < 0 or ax0_0 == 0) and ax0_0 * 2 + ax0 < seq_len) T.reads(NT_matmul_pad_local[0, v0, v1]) T.writes(NT_matmul[0, v0, v1]) NT_matmul[0, v0, v1] = NT_matmul_pad_local[0, v0, v1] else: if T.tvm_thread_invariant(seq_len <= 8): with T.block("root"): T.reads() T.writes() NT_matmul_pad_local = T.alloc_buffer((1, (seq_len + 3) // 4 * 4, 2048), "float16", scope="local") NT_matmul_pad_rf_local = T.alloc_buffer((256, 1, (seq_len + 3) // 4 * 4, 2048), "float16", scope="local") NT_matmul_pad_rf_local_1 = T.alloc_buffer((32, 1, (seq_len + 3) // 4 * 4, 2048), "float16", scope="local") for ax0_0 in T.thread_binding((seq_len + 3) // 4, thread="blockIdx.y"): for ax1_fused_0 in T.thread_binding(512, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(2, thread="threadIdx.x"): for ax2_fused_1_ax2_fused_3_fused_0 in T.thread_binding(32, thread="threadIdx.y"): for ax0_1_init, ax1_fused_2_init in T.grid(4, 2): for ax2_fused_1_ax2_fused_3_fused_1_init in T.vectorized(8): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1_init) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax0_1_init) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2_init) T.reads() T.writes(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = T.float16(0.0) for ax2_fused_0 in T.serial(64, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax0_1, ax1_fused_2, ax2_fused_2 in T.grid(4, 2, 1): for ax2_fused_1_ax2_fused_3_fused_1 in T.vectorized(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused = T.axis.spatial(256, ax2_fused_1_ax2_fused_3_fused_0 * 8 + ax2_fused_1_ax2_fused_3_fused_1) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax0_1) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_1 * 2 + ax1_fused_2) vax2_fused_2, vax2_fused_0 = T.axis.remap("RR", [ax2_fused_2, ax2_fused_0]) T.reads(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1], mul39[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], model_layers_0_mlp_down_proj_weight3[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused]) T.writes(NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1]) NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] = NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused, 0, v0, v1] + T.if_then_else(v0 < seq_len, mul39[0, v0, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused], T.float16(0.0)) * model_layers_0_mlp_down_proj_weight3[v1, vax2_fused_2 * 8 + vax2_fused_0 * 256 + vax2_fused_1_ax2_fused_3_fused] for ax3_fused_0_ax3_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): for ax3_fused_2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 8, "pragma_unroll_explicit": 1}): for ax2 in range(4): for ax3_fused_2_1 in T.vectorized(2): with T.block("NT_matmul_rf_init"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.spatial(32, ax0) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads() T.writes(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = T.float16(0.0) for ax1 in range(8): with T.block("NT_matmul_rf_update"): vax2_fused_1_ax2_fused_3_fused_0, vax2_fused_1_ax2_fused_3_fused_1 = T.axis.remap("SR", [ax0, ax1]) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax2) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax3_fused_0_ax3_fused_1_fused * 2 + ax3_fused_2_0 * 2 + ax3_fused_2_1) T.reads(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1], NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1]) T.writes(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] = NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] + NT_matmul_pad_rf_local[vax2_fused_1_ax2_fused_3_fused_0 * 8 + vax2_fused_1_ax2_fused_3_fused_1, 0, v0, v1] for ax2_fused_2, ax1 in T.grid(2, 4): for ax2_fused_0_ax2_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax0 in T.thread_binding(32, thread="threadIdx.y"): with T.block("NT_matmul"): vax2_fused_1_ax2_fused_3_fused_0 = T.axis.reduce(32, ax0) v0 = T.axis.spatial((seq_len + 3) // 4 * 4, ax0_0 * 4 + ax1) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax2_fused_0_ax2_fused_1_fused * 2 + ax2_fused_2) T.reads(NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1]) T.writes(NT_matmul_pad_local[0, v0, v1]) with T.init(): NT_matmul_pad_local[0, v0, v1] = T.float16(0.0) NT_matmul_pad_local[0, v0, v1] = NT_matmul_pad_local[0, v0, v1] + NT_matmul_pad_rf_local_1[vax2_fused_1_ax2_fused_3_fused_0, 0, v0, v1] for ax0 in range(4): for ax1_fused_0_ax1_fused_1_fused in T.thread_binding(2, thread="threadIdx.x"): for ax1_fused_2 in range(2): with T.block("NT_matmul_pad"): v0 = T.axis.spatial(seq_len, ax0_0 * 4 + ax0) v1 = T.axis.spatial(2048, ax1_fused_0 * 4 + ax1_fused_0_ax1_fused_1_fused * 2 + ax1_fused_2) T.where((ax0_0 - (seq_len + 3) // 4 < 0 or ax0_0 == 0) and ax0_0 * 4 + ax0 < seq_len) T.reads(NT_matmul_pad_local[0, v0, v1]) T.writes(NT_matmul[0, v0, v1]) NT_matmul[0, v0, v1] = NT_matmul_pad_local[0, v0, v1] else: with T.block("root"): T.reads() T.writes() NT_matmul_reindex_pad_local = T.alloc_buffer((1, (seq_len + 7) // 8 * 8, 2048), "float16", scope="local") for ax0_ax1_0_fused in T.thread_binding((seq_len + 7) // 8, thread="blockIdx.y"): for ax2_0 in T.thread_binding(8, thread="blockIdx.x"): for ax1_1 in T.thread_binding(1, thread="vthread.y"): for ax2_1 in T.thread_binding(1, thread="vthread.x"): for ax1_2 in T.thread_binding(4, thread="threadIdx.y"): for ax2_2 in T.thread_binding(32, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 16, "pragma_unroll_explicit": 1}): for ax1_3_init, ax2_3_0_init in T.grid(2, 1): for ax2_3_1_init in T.vectorized(8): with T.block("NT_matmul_init"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3_init) v2 = T.axis.spatial(2048, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0_init * 8 + ax2_3_1_init) T.reads() T.writes(NT_matmul_reindex_pad_local[0, v1, v2]) NT_matmul_reindex_pad_local[0, v1, v2] = T.float16(0.0) for ax3_0, ax3_1, ax1_3, ax2_3_0 in T.grid(1024, 16, 2, 1): for ax2_3_1 in T.vectorized(8): with T.block("NT_matmul_update"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3) v2 = T.axis.spatial(2048, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0 * 8 + ax2_3_1) v3 = T.axis.reduce(16384, ax3_0 * 16 + ax3_1) T.reads(NT_matmul_reindex_pad_local[0, v1, v2], mul39[0, v1, v3], model_layers_0_mlp_down_proj_weight3[v2, v3]) T.writes(NT_matmul_reindex_pad_local[0, v1, v2]) NT_matmul_reindex_pad_local[0, v1, v2] = NT_matmul_reindex_pad_local[0, v1, v2] + T.if_then_else(v1 < seq_len, mul39[0, v1, v3], T.float16(0.0)) * model_layers_0_mlp_down_proj_weight3[v2, v3] for ax0, ax1, ax2_0_1 in T.grid(1, 2, 1): for ax2_1_1 in T.vectorized(8): with T.block("NT_matmul_reindex_pad_local"): v0 = T.axis.spatial(1, ax0) v1 = T.axis.spatial((seq_len + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1) v2 = T.axis.spatial(2048, ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1) T.where(ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1 < seq_len) T.reads(NT_matmul_reindex_pad_local[v0, v1, v2]) T.writes(NT_matmul[0, v1, v2]) NT_matmul[0, v1, v2] = NT_matmul_reindex_pad_local[v0, v1, v2] @T.prim_func def gemma_q0f16_NT_matmul9(var_take1: T.handle, var_model_embed_tokens_weight3: T.handle, var_NT_matmul: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() take1 = T.match_buffer(var_take1, (1, batch_size, 2048), "float16") vocab_size = T.int32() model_embed_tokens_weight3 = T.match_buffer(var_model_embed_tokens_weight3, (vocab_size, 2048), "float16") NT_matmul = T.match_buffer(var_NT_matmul, (1, batch_size, vocab_size)) # with T.block("root"): NT_matmul_reindex_pad_local = T.alloc_buffer((1, (batch_size + 7) // 8 * 8, (vocab_size + 255) // 256 * 256), scope="local") for ax0_ax1_0_fused in T.thread_binding((batch_size + 7) // 8, thread="blockIdx.y"): for ax2_0 in T.thread_binding((vocab_size + 255) // 256, thread="blockIdx.x"): for ax1_1 in T.thread_binding(1, thread="vthread.y"): for ax2_1 in T.thread_binding(1, thread="vthread.x"): for ax1_2 in T.thread_binding(4, thread="threadIdx.y"): for ax2_2 in T.thread_binding(32, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 16, "pragma_unroll_explicit": 1}): for ax1_3_init, ax2_3_0_init in T.grid(2, 1): for ax2_3_1_init in T.vectorized(8): with T.block("NT_matmul_init"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3_init) v2 = T.axis.spatial((vocab_size + 255) // 256 * 256, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0_init * 8 + ax2_3_1_init) T.reads() T.writes(NT_matmul_reindex_pad_local[0, v1, v2]) NT_matmul_reindex_pad_local[0, v1, v2] = T.float32(0.0) for ax3_0, ax3_1, ax1_3, ax2_3_0 in T.grid(128, 16, 2, 1): for ax2_3_1 in T.vectorized(8): with T.block("NT_matmul_update"): v0 = T.axis.spatial(1, 0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_1 * 8 + ax1_2 * 2 + ax1_3) v2 = T.axis.spatial((vocab_size + 255) // 256 * 256, ax2_0 * 256 + ax2_1 * 256 + ax2_2 * 8 + ax2_3_0 * 8 + ax2_3_1) v3 = T.axis.reduce(2048, ax3_0 * 16 + ax3_1) T.reads(NT_matmul_reindex_pad_local[0, v1, v2], take1[0, v1, v3], model_embed_tokens_weight3[v2, v3]) T.writes(NT_matmul_reindex_pad_local[0, v1, v2]) NT_matmul_reindex_pad_local[0, v1, v2] = NT_matmul_reindex_pad_local[0, v1, v2] + T.Cast("float32", T.if_then_else(v1 < batch_size, take1[0, v1, v3], T.float16(0.0))) * T.Cast("float32", T.if_then_else(v2 < vocab_size, model_embed_tokens_weight3[v2, v3], T.float16(0.0))) for ax0, ax1, ax2_0_1 in T.grid(1, 2, 1): for ax2_1_1 in T.vectorized(8): with T.block("NT_matmul_reindex_pad_local"): v0 = T.axis.spatial(1, ax0) v1 = T.axis.spatial((batch_size + 7) // 8 * 8, ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1) v2 = T.axis.spatial((vocab_size + 255) // 256 * 256, ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1) T.where(ax0_ax1_0_fused * 8 + ax1_2 * 2 + ax1 < batch_size and ax2_0 * 256 + ax2_2 * 8 + ax2_0_1 * 8 + ax2_1_1 < vocab_size) T.reads(NT_matmul_reindex_pad_local[v0, v1, v2]) T.writes(NT_matmul[0, v1, v2]) NT_matmul[0, v1, v2] = NT_matmul_reindex_pad_local[v0, v1, v2] @T.prim_func def gemma_q0f16_apply_bitmask_inplace(var_logits: T.handle, var_seq_ids: T.handle, var_bitmask: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": T.bool(True), "tir.noalias": T.bool(True)}) batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True) logits = T.match_buffer(var_logits, (batch_size, vocab_size)) num_seq = T.int32(is_size_var=True) seq_ids = T.match_buffer(var_seq_ids, (num_seq,), "int32") bitmask = T.match_buffer(var_bitmask, (batch_size, (vocab_size + 31) // 32), "int32") # with T.block("root"): for fused_s_v_0 in T.thread_binding((num_seq * vocab_size + 255) // 256, thread="blockIdx.x"): for fused_s_v_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("block"): vs = T.axis.spatial(num_seq, (fused_s_v_0 * 256 + fused_s_v_1) // vocab_size) vv = T.axis.spatial(vocab_size, (fused_s_v_0 * 256 + fused_s_v_1) % vocab_size) T.where(fused_s_v_0 * 256 + fused_s_v_1 < num_seq * vocab_size) T.reads(bitmask[seq_ids[vs], vv // 32], seq_ids[vs], logits[seq_ids[vs], vv]) T.writes(logits[seq_ids[vs], vv]) logits[seq_ids[vs], vv] = T.if_then_else(T.bitwise_and(T.shift_right(bitmask[seq_ids[vs], vv // 32], vv % 32), 1) == 1, logits[seq_ids[vs], vv], T.float32(-340282346638528859811704183484516925440.0)) @T.prim_func def gemma_q0f16_apply_logit_bias_inplace(var_logits: T.handle, var_pos2seq_id: T.handle, var_token_ids: T.handle, var_logit_bias: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": T.bool(True), "tir.noalias": T.bool(True)}) batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True) logits = T.match_buffer(var_logits, (batch_size, vocab_size)) num_token = T.int32(is_size_var=True) pos2seq_id = T.match_buffer(var_pos2seq_id, (num_token,), "int32") token_ids = T.match_buffer(var_token_ids, (num_token,), "int32") logit_bias = T.match_buffer(var_logit_bias, (num_token,)) # with T.block("root"): for p0 in T.thread_binding((num_token + 255) // 256, thread="blockIdx.x"): for p1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("block"): vp = T.axis.spatial(num_token, p0 * 256 + p1) T.where(p0 * 256 + p1 < num_token) T.reads(logits[pos2seq_id[vp], token_ids[vp]], pos2seq_id[vp], token_ids[vp], logit_bias[vp]) T.writes(logits[pos2seq_id[vp], token_ids[vp]]) logits[pos2seq_id[vp], token_ids[vp]] = logits[pos2seq_id[vp], token_ids[vp]] + logit_bias[vp] @T.prim_func def gemma_q0f16_apply_penalty_inplace(var_logits: T.handle, var_seq_ids: T.handle, var_pos2seq_id: T.handle, var_token_ids: T.handle, var_token_cnt: T.handle, var_penalties: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": T.bool(True), "tir.noalias": T.bool(True)}) batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True) logits = T.match_buffer(var_logits, (batch_size, vocab_size)) num_seq = T.int32(is_size_var=True) seq_ids = T.match_buffer(var_seq_ids, (num_seq,), "int32") num_token = T.int32(is_size_var=True) pos2seq_id = T.match_buffer(var_pos2seq_id, (num_token,), "int32") token_ids = T.match_buffer(var_token_ids, (num_token,), "int32") token_cnt = T.match_buffer(var_token_cnt, (num_token,), "int32") penalties = T.match_buffer(var_penalties, (num_seq, 3)) # with T.block("root"): for p0 in T.thread_binding((num_token + 255) // 256, thread="blockIdx.x"): for p1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("block"): vp = T.axis.spatial(num_token, p0 * 256 + p1) T.where(p0 * 256 + p1 < num_token) T.reads(logits[seq_ids[pos2seq_id[vp]], token_ids[vp]], seq_ids[pos2seq_id[vp]], pos2seq_id[vp], token_ids[vp], penalties[pos2seq_id[vp], 0:3], token_cnt[vp]) T.writes(logits[seq_ids[pos2seq_id[vp]], token_ids[vp]]) logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] = logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] - (penalties[pos2seq_id[vp], 0] + T.Cast("float32", token_cnt[vp]) * penalties[pos2seq_id[vp], 1]) logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] = T.if_then_else(logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] < T.float32(0.0), logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] * penalties[pos2seq_id[vp], 2], logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] / penalties[pos2seq_id[vp], 2]) @T.prim_func def gemma_q0f16_batch_decode_paged_kv(_0: T.int32, Q_handle: T.handle, pages_handle: T.handle, page_table_indptr_handle: T.handle, page_table_values_handle: T.handle, var_length_info: T.handle, k_rope_pos_offset_handle: T.handle, q_rope_position_handle: T.handle, output_handle: T.handle, lse_handle: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) B = T.int32(is_size_var=True) Q = T.match_buffer(Q_handle, (B, 8, 256), "float16") max_num_pages = T.int32(is_size_var=True) pages = T.match_buffer(pages_handle, (max_num_pages, 2, 1, 16, 256), "float16", offset_factor=1) page_table_indptr = T.match_buffer(page_table_indptr_handle, (B + 1,), "int32", offset_factor=1) nnz_pages = T.int32(is_size_var=True) page_table_values = T.match_buffer(page_table_values_handle, (nnz_pages,), "int32", offset_factor=1) length_info = T.match_buffer(var_length_info, (B,), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(k_rope_pos_offset_handle, (B,), "int32", offset_factor=1) q_rope_position = T.match_buffer(q_rope_position_handle, (B,), "int32", offset_factor=1) output = T.match_buffer(output_handle, (B, 8, 256), "float16") lse = T.match_buffer(lse_handle, (B, 8)) # with T.block("root"): sm_scale: T.float32 = T.float32(0.090168440055560212) for bx in T.thread_binding(B, thread="blockIdx.x"): for fused_by_bz in T.thread_binding(2, thread="blockIdx.y"): for ty in T.thread_binding(4, thread="threadIdx.y"): for tx in T.thread_binding(64, thread="threadIdx.x"): for tz in T.thread_binding(1, thread="threadIdx.z"): with T.block("attn"): T.reads(page_table_indptr[bx:bx + 2], length_info[bx], q_rope_position[bx], Q[bx, fused_by_bz * 4 + ty, tx * 4 - 128:tx * 4 - 128 + 260]) T.writes(output[bx, T.Mul(0, 8) + fused_by_bz * 4 + ty, tx * 4:tx * 4 + 4], lse[bx, T.Mul(0, 8) + fused_by_bz * 4 + ty]) Q_local = T.alloc_buffer((4,), "float16", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") K_smem = T.alloc_buffer((4, 256), "float16", scope="shared") V_smem = T.alloc_buffer((4, 256), "float16", scope="shared") O_allreduce = T.alloc_buffer((1, 4, 256), scope="shared") md_allreduce = T.alloc_buffer((1, 4, 2), scope="shared") S_reduce_local = T.alloc_buffer((1,), scope="local") t0 = T.alloc_buffer((1,), scope="local") S_local = T.alloc_buffer((4,), scope="local") QK_local = T.alloc_buffer((4,), scope="local") V_local = T.alloc_buffer((4,), "float16", scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_prev = T.alloc_buffer((1,), scope="local") other_m = T.alloc_buffer((1,), scope="local") other_d = T.alloc_buffer((1,), scope="local") exp_mprev = T.alloc_buffer((1,), scope="local") exp_otherm = T.alloc_buffer((1,), scope="local") other_o = T.alloc_buffer((4,), scope="local") st_m = T.alloc_buffer((1,), scope="local") st_d = T.alloc_buffer((1,), scope="local") O_local = T.alloc_buffer((4,), scope="local") by: T.int32 = 0 bz: T.int32 = fused_by_bz batch_idx: T.int32 = bx cur_page_indptr_begin: T.int32 = page_table_indptr[batch_idx] cur_page_indptr_end: T.int32 = page_table_indptr[batch_idx + 1] kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[batch_idx], 0) st_m[0] = T.float32(-50000.0) st_d[0] = T.float32(1.0) for vec in T.vectorized(4): O_local[vec] = T.float32(0.0) for vec in T.vectorized(4): freq = T.float32() Q_local[vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", Q[bx, by * 8 + bz * 4 + ty, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 128, Q[bx, by * 8 + bz * 4 + ty, tx * 4 + vec + 128] * T.float16(-1.0), Q[bx, by * 8 + bz * 4 + ty, tx * 4 + vec - 128]))), where={freq: T.Cast("float32", q_rope_position[batch_idx]) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 256) / T.float32(256.0))}), Q[bx, by * 8 + bz * 4 + ty, tx * 4 + vec]) for iterator in range((kv_chunk_len[0] + 3) // 4): tile_start_s: T.int32 = tz * 4 + ty tile_start_g: T.int32 = (iterator + tz) * 4 + ty for j in range(1): with T.block("KV_load"): T.reads() T.writes() row_g: T.int32 = tile_start_g + j if row_g < kv_chunk_len[0]: seq_offset: T.int32 = row_g page_no: T.int32 = page_table_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 for vec in T.vectorized(4): freq = T.float32() K_smem[tile_start_s + j, tx * 4 + vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 128, pages[page_no, 0, by, page_offset, tx * 4 + vec + 128] * T.float16(-1.0), pages[page_no, 0, by, page_offset, tx * 4 + vec - 128]))), where={freq: T.Cast("float32", k_rope_pos_offset[batch_idx] + row_g) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 256) / T.float32(256.0))}), pages[page_no, 0, by, page_offset, tx * 4 + vec]) V_smem[tile_start_s + j, tx * 4 + vec] = pages[page_no, 1, by, page_offset, tx * 4 + vec] else: for vec in T.vectorized(4): K_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0) V_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0) T.tvm_storage_sync("shared") m_prev[0] = st_m[0] for j in range(4): for vec in T.vectorized(4): QK_local[vec] = T.Cast("float32", Q_local[vec]) * T.Cast("float32", K_smem[tz * 4 + j, tx * 4 + vec]) * attn_score_scaling_factor * sm_scale S_reduce_local[0] = T.float32(0.0) for vec in T.unroll(4): S_reduce_local[0] = S_reduce_local[0] + QK_local[vec] with T.block("block_cross_thread"): T.reads(S_reduce_local[0]) T.writes(t0[0]) T.attr(T.comm_reducer(lambda x0, y0: x0 + y0, [T.float32(0.0)]), "reduce_scope", T.reinterpret("handle", T.uint64(0))) T.tvm_thread_allreduce(T.uint32(1), S_reduce_local[0], T.bool(True), t0[0], tx) S_local[j] = T.float32(-50000.0) if (iterator + tz) * 4 + j < kv_chunk_len[0]: S_local[j] = t0[0] st_m[0] = T.max(st_m[0], S_local[j]) o_scale: T.float32 = T.exp2(m_prev[0] - st_m[0]) st_d[0] = st_d[0] * o_scale for j in range(4): S_local[j] = T.exp2(S_local[j] - st_m[0]) st_d[0] = st_d[0] + S_local[j] for j in T.vectorized(4): O_local[j] = O_local[j] * o_scale for j in range(4): for vec in T.vectorized(4): V_local[vec] = V_smem[tz * 4 + j, tx * 4 + vec] for vec in T.vectorized(4): O_local[vec] = O_local[vec] + T.Cast("float32", V_local[vec]) * S_local[j] for vec in T.vectorized(4): O_local[vec] = O_local[vec] / st_d[0] for vec in T.vectorized(4): output[batch_idx, by * 8 + bz * 4 + ty, tx * 4 + vec] = T.Cast("float16", O_local[vec]) lse[batch_idx, by * 8 + bz * 4 + ty] = st_m[0] + T.log2(st_d[0]) @T.prim_func def gemma_q0f16_batch_decode_paged_kv_sliding_window(_0: T.int32, Q_handle: T.handle, pages_handle: T.handle, page_table_indptr_handle: T.handle, page_table_values_handle: T.handle, var_length_info: T.handle, k_rope_pos_offset_handle: T.handle, q_rope_position_handle: T.handle, output_handle: T.handle, lse_handle: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) B = T.int32(is_size_var=True) Q = T.match_buffer(Q_handle, (B, 8, 256), "float16") max_num_pages = T.int32(is_size_var=True) pages = T.match_buffer(pages_handle, (max_num_pages, 2, 1, 16, 256), "float16", offset_factor=1) page_table_indptr = T.match_buffer(page_table_indptr_handle, (B + 1,), "int32", offset_factor=1) nnz_pages = T.int32(is_size_var=True) page_table_values = T.match_buffer(page_table_values_handle, (nnz_pages,), "int32", offset_factor=1) length_info = T.match_buffer(var_length_info, (3, B), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(k_rope_pos_offset_handle, (B,), "int32", offset_factor=1) q_rope_position = T.match_buffer(q_rope_position_handle, (B,), "int32", offset_factor=1) output = T.match_buffer(output_handle, (B, 8, 256), "float16") lse = T.match_buffer(lse_handle, (B, 8)) # with T.block("root"): sm_scale: T.float32 = T.float32(0.090168440055560212) for bx in T.thread_binding(B, thread="blockIdx.x"): for fused_by_bz in T.thread_binding(2, thread="blockIdx.y"): for ty in T.thread_binding(4, thread="threadIdx.y"): for tx in T.thread_binding(64, thread="threadIdx.x"): for tz in T.thread_binding(1, thread="threadIdx.z"): with T.block("attn"): T.reads(page_table_indptr[bx:bx + 2], length_info[0:3, bx], q_rope_position[bx], Q[bx, fused_by_bz * 4 + ty, tx * 4 - 128:tx * 4 - 128 + 260]) T.writes(output[bx, T.Mul(0, 8) + fused_by_bz * 4 + ty, tx * 4:tx * 4 + 4], lse[bx, T.Mul(0, 8) + fused_by_bz * 4 + ty]) Q_local = T.alloc_buffer((4,), "float16", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") K_smem = T.alloc_buffer((4, 256), "float16", scope="shared") V_smem = T.alloc_buffer((4, 256), "float16", scope="shared") O_allreduce = T.alloc_buffer((1, 4, 256), scope="shared") md_allreduce = T.alloc_buffer((1, 4, 2), scope="shared") S_reduce_local = T.alloc_buffer((1,), scope="local") t0 = T.alloc_buffer((1,), scope="local") S_local = T.alloc_buffer((4,), scope="local") QK_local = T.alloc_buffer((4,), scope="local") V_local = T.alloc_buffer((4,), "float16", scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_prev = T.alloc_buffer((1,), scope="local") other_m = T.alloc_buffer((1,), scope="local") other_d = T.alloc_buffer((1,), scope="local") exp_mprev = T.alloc_buffer((1,), scope="local") exp_otherm = T.alloc_buffer((1,), scope="local") other_o = T.alloc_buffer((4,), scope="local") st_m = T.alloc_buffer((1,), scope="local") st_d = T.alloc_buffer((1,), scope="local") O_local = T.alloc_buffer((4,), scope="local") by: T.int32 = 0 bz: T.int32 = fused_by_bz batch_idx: T.int32 = bx cur_page_indptr_begin: T.int32 = page_table_indptr[batch_idx] cur_page_indptr_end: T.int32 = page_table_indptr[batch_idx + 1] kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[0, batch_idx] - length_info[1, batch_idx] + length_info[2, batch_idx], 0) st_m[0] = T.float32(-50000.0) st_d[0] = T.float32(1.0) for vec in T.vectorized(4): O_local[vec] = T.float32(0.0) for vec in T.vectorized(4): freq = T.float32() Q_local[vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", Q[bx, by * 8 + bz * 4 + ty, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 128, Q[bx, by * 8 + bz * 4 + ty, tx * 4 + vec + 128] * T.float16(-1.0), Q[bx, by * 8 + bz * 4 + ty, tx * 4 + vec - 128]))), where={freq: T.Cast("float32", q_rope_position[batch_idx]) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 256) / T.float32(256.0))}), Q[bx, by * 8 + bz * 4 + ty, tx * 4 + vec]) for iterator in range((kv_chunk_len[0] + 3) // 4): tile_start_s: T.int32 = tz * 4 + ty tile_start_g: T.int32 = (iterator + tz) * 4 + ty for j in range(1): with T.block("KV_load"): T.reads() T.writes() row_g: T.int32 = tile_start_g + j if row_g < kv_chunk_len[0]: seq_offset: T.int32 = T.if_then_else(row_g < length_info[2, batch_idx], row_g, row_g - length_info[2, batch_idx] + length_info[1, batch_idx]) page_no: T.int32 = page_table_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 for vec in T.vectorized(4): freq = T.float32() K_smem[tile_start_s + j, tx * 4 + vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 128, pages[page_no, 0, by, page_offset, tx * 4 + vec + 128] * T.float16(-1.0), pages[page_no, 0, by, page_offset, tx * 4 + vec - 128]))), where={freq: T.Cast("float32", k_rope_pos_offset[batch_idx] + row_g) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 256) / T.float32(256.0))}), pages[page_no, 0, by, page_offset, tx * 4 + vec]) V_smem[tile_start_s + j, tx * 4 + vec] = pages[page_no, 1, by, page_offset, tx * 4 + vec] else: for vec in T.vectorized(4): K_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0) V_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0) T.tvm_storage_sync("shared") m_prev[0] = st_m[0] for j in range(4): for vec in T.vectorized(4): QK_local[vec] = T.Cast("float32", Q_local[vec]) * T.Cast("float32", K_smem[tz * 4 + j, tx * 4 + vec]) * attn_score_scaling_factor * sm_scale S_reduce_local[0] = T.float32(0.0) for vec in T.unroll(4): S_reduce_local[0] = S_reduce_local[0] + QK_local[vec] with T.block("block_cross_thread"): T.reads(S_reduce_local[0]) T.writes(t0[0]) T.attr(T.comm_reducer(lambda x0, y0: x0 + y0, [T.float32(0.0)]), "reduce_scope", T.reinterpret("handle", T.uint64(0))) T.tvm_thread_allreduce(T.uint32(1), S_reduce_local[0], T.bool(True), t0[0], tx) S_local[j] = T.float32(-50000.0) if (iterator + tz) * 4 + j < kv_chunk_len[0]: S_local[j] = t0[0] st_m[0] = T.max(st_m[0], S_local[j]) o_scale: T.float32 = T.exp2(m_prev[0] - st_m[0]) st_d[0] = st_d[0] * o_scale for j in range(4): S_local[j] = T.exp2(S_local[j] - st_m[0]) st_d[0] = st_d[0] + S_local[j] for j in T.vectorized(4): O_local[j] = O_local[j] * o_scale for j in range(4): for vec in T.vectorized(4): V_local[vec] = V_smem[tz * 4 + j, tx * 4 + vec] for vec in T.vectorized(4): O_local[vec] = O_local[vec] + T.Cast("float32", V_local[vec]) * S_local[j] for vec in T.vectorized(4): O_local[vec] = O_local[vec] / st_d[0] for vec in T.vectorized(4): output[batch_idx, by * 8 + bz * 4 + ty, tx * 4 + vec] = T.Cast("float16", O_local[vec]) lse[batch_idx, by * 8 + bz * 4 + ty] = st_m[0] + T.log2(st_d[0]) @T.prim_func def gemma_q0f16_batch_prefill_paged_kv(_0: T.int32, var_q: T.handle, var_q_indptr: T.handle, var_pages: T.handle, var_page_indptr: T.handle, var_page_values: T.handle, var_length_info: T.handle, var_k_rope_pos_offset: T.handle, var_q_rope_position: T.handle, var_output: T.handle, var_lse: T.handle, causal: T.int32, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) total_len = T.int32(is_size_var=True) q = T.match_buffer(var_q, (total_len, 8, 256), "float16") batch_size = T.int32(is_size_var=True) q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) max_num_pages = T.int32(is_size_var=True) pages = T.match_buffer(var_pages, (max_num_pages, 2, 1, 16, 256), "float16", offset_factor=1) page_indptr = T.match_buffer(var_page_indptr, (batch_size + 1,), "int32", offset_factor=1) nnz_pages = T.int32(is_size_var=True) page_values = T.match_buffer(var_page_values, (nnz_pages,), "int32", offset_factor=1) length_info = T.match_buffer(var_length_info, (batch_size,), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1) q_rope_position = T.match_buffer(var_q_rope_position, (total_len,), "int32", offset_factor=1) output = T.match_buffer(var_output, (total_len, 8, 256), "float16") lse = T.match_buffer(var_lse, (total_len, 8)) # with T.block("root"): for lbx in T.thread_binding(16, thread="blockIdx.x"): for lby in T.thread_binding(1, thread="blockIdx.y"): for lty in T.thread_binding(4, thread="threadIdx.y"): for ltx in T.thread_binding(32, thread="threadIdx.x"): with T.block("attn"): bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) T.reads() T.writes() tile_id = T.alloc_buffer((1,), "int32", scope="local") batch_idx = T.alloc_buffer((1,), "int32", scope="local") batch_tiles = T.alloc_buffer((1,), "int32", scope="local") batch_rows = T.alloc_buffer((1,), "int32", scope="local") iterator = T.alloc_buffer((1,), "int32", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") Q_smem = T.alloc_buffer((16, 256), "float16", scope="shared") K_smem = T.alloc_buffer((16, 256), "float16", scope="shared") V_smem = T.alloc_buffer((16, 256), "float16", scope="shared") S_smem = T.alloc_buffer((16, 16), scope="shared") S_local = T.alloc_buffer((16, 16), scope="local") O_local = T.alloc_buffer((16, 256), scope="local") m_smem = T.alloc_buffer((16,), scope="shared") m_prev_smem = T.alloc_buffer((16,), scope="shared") d_smem = T.alloc_buffer((16,), scope="shared") m_new = T.alloc_buffer((1,), scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_new = T.alloc_buffer((1,), scope="local") tile_id[0] = bx batch_idx[0] = 0 batch_rows[0] = (q_indptr[1] - q_indptr[0]) * 8 batch_tiles[0] = (batch_rows[0] + 16 - 1) // 16 while T.tvm_thread_invariant(batch_idx[0] < batch_size): while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: tile_id[0] = tile_id[0] - batch_tiles[0] batch_idx[0] = batch_idx[0] + 1 if batch_idx[0] < batch_size: b_idx: T.int32 = batch_idx[0] batch_rows[0] = (q_indptr[b_idx + 1] - q_indptr[b_idx]) * 8 batch_tiles[0] = (batch_rows[0] + 16 - 1) // 16 if T.tvm_thread_invariant(batch_idx[0] < batch_size): b_idx: T.int32 = batch_idx[0] LH_start: T.int32 = tile_id[0] * 16 q_indptr_val: T.int32 = q_indptr[b_idx] cur_page_indptr_begin: T.int32 = page_indptr[b_idx] cur_page_indptr_end: T.int32 = page_indptr[b_idx + 1] kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[b_idx], 0) T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: m_smem[row] = T.float32(-50000.0) d_smem[row] = T.float32(1.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 8): with T.block("O_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) T.reads() T.writes(O_local[i, j]) O_local[i, j] = T.float32(0.0) T.tvm_storage_sync("shared") for li_lj_fused_0 in range(8): for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for li_lj_fused_3 in T.vectorized(4): with T.block("Q_load"): i = T.axis.spatial(16, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 256) j = T.axis.spatial(256, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = q_indptr_val + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: freq = T.float32() Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 128, q[cur_L, cur_H_qo, j + 128] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 128]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 256) / T.float32(256.0))}), q[cur_L, cur_H_qo, j]) else: Q_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for iterator_1 in range((kv_chunk_len[0] + 15) // 16): L_kv_start: T.int32 = iterator_1 * 16 for lz_ly_fused_0 in range(8): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("K_load"): i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 256) j = T.axis.spatial(256, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = cur_L page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 freq = T.float32() K_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 128, pages[page_no, 0, by, page_offset, j + 128] * T.float16(-1.0), pages[page_no, 0, by, page_offset, j - 128]))), where={freq: T.Cast("float32", k_rope_pos_offset[b_idx] + cur_L) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 256) / T.float32(256.0))}), pages[page_no, 0, by, page_offset, j]) else: K_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for lz_ly_fused_0 in range(8): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("V_load"): i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 256) j = T.axis.spatial(256, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = cur_L page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 V_smem[i, j] = pages[page_no, 1, by, page_offset, j] else: V_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") with T.block(""): T.reads(Q_smem[0:16, 0:256], K_smem[0:16, 0:256]) T.writes(S_local[0:16, 0:16]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(1, 2): with T.block("S_gemm_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 + li_1_init) j = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 2 + lj_1_init) T.reads() T.writes(S_local[i, j]) S_local[i, j] = T.float32(0.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, li_1, lj_1, lk_1 in T.grid(32, 1, 2, 8): with T.block("S_gemm_update"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 + li_1) j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1) k = T.axis.reduce(256, lk_0 * 8 + lk_1) T.reads(S_local[i, j], Q_smem[i, k], K_smem[j, k]) T.writes(S_local[i, j]) S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k]) * T.Cast("float32", K_smem[j, k]) * attn_score_scaling_factor * T.float32(0.090168440055560212) T.tvm_storage_sync("shared") for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(1, 2): with T.block("S_store"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 + li_1) j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1) T.reads(S_local[i, j]) T.writes(S_smem[i, j]) S_smem[i, j] = S_local[i, j] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: with T.block("update1"): T.reads(m_smem[row], kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:16], d_smem[row], m_prev[i]) T.writes(m_prev[i], m_new[i], d_new[i]) m_prev[i] = m_smem[row] m_new[i] = m_smem[row] row_: T.int32 = (LH_start + row) // 8 for j in range(16): if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): m_new[i] = T.max(m_new[i], S_smem[row, j]) d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx with T.block("update"): T.reads(kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:16], m_new[i]) T.writes(S_smem[row, 0:16]) for j in range(16): if row < 16: row_: T.int32 = (LH_start + row) // 8 if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) else: S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: with T.block("update"): T.reads(d_new[i], S_smem[row, 0:16], m_new[i], m_prev[i]) T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) for j in range(16): d_new[i] = d_new[i] + S_smem[row, j] m_smem[row] = m_new[i] d_smem[row] = d_new[i] m_prev_smem[row] = m_prev[i] T.tvm_storage_sync("shared") with T.block(""): T.reads(m_prev_smem[0:16], m_smem[0:16], S_smem[0:16, 0:16], V_smem[0:16, 0:256]) T.writes(O_local[0:16, 0:256]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(4, 8): with T.block("O_gemm_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 32 * 4 + li_1_init) j = T.axis.spatial(256, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 32 * 8 + lj_1_init) T.reads() T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, lk_1, li_1, lj_1 in T.grid(2, 8, 4, 8): with T.block("O_gemm_update"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) k = T.axis.reduce(16, lk_0 * 8 + lk_1) T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k], V_smem[k, j]) T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] + S_smem[i, k] * T.Cast("float32", V_smem[k, j]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 8): with T.block("O_store"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) T.writes(output[q_indptr[b_idx] + (LH_start + i) // 8, by * 8 + (LH_start + i) % 8, j]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) for li_0 in range(1): for li_1 in T.thread_binding(4, thread="threadIdx.y"): for li_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("lse_store"): i = T.axis.spatial(16, li_0 * 128 + li_1 * 32 + li_2) T.where((li_0 * 4 + li_1) * 32 + li_2 < 16) T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) T.writes(lse[q_indptr[b_idx] + (LH_start + i) // 8, by * 8 + (LH_start + i) % 8]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) tile_id[0] = tile_id[0] + 16 @T.prim_func def gemma_q0f16_batch_prefill_paged_kv_sliding_window(_0: T.int32, var_q: T.handle, var_q_indptr: T.handle, var_pages: T.handle, var_page_indptr: T.handle, var_page_values: T.handle, var_length_info: T.handle, var_k_rope_pos_offset: T.handle, var_q_rope_position: T.handle, var_output: T.handle, var_lse: T.handle, causal: T.int32, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) total_len = T.int32(is_size_var=True) q = T.match_buffer(var_q, (total_len, 8, 256), "float16") batch_size = T.int32(is_size_var=True) q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) max_num_pages = T.int32(is_size_var=True) pages = T.match_buffer(var_pages, (max_num_pages, 2, 1, 16, 256), "float16", offset_factor=1) page_indptr = T.match_buffer(var_page_indptr, (batch_size + 1,), "int32", offset_factor=1) nnz_pages = T.int32(is_size_var=True) page_values = T.match_buffer(var_page_values, (nnz_pages,), "int32", offset_factor=1) length_info = T.match_buffer(var_length_info, (3, batch_size), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1) q_rope_position = T.match_buffer(var_q_rope_position, (total_len,), "int32", offset_factor=1) output = T.match_buffer(var_output, (total_len, 8, 256), "float16") lse = T.match_buffer(var_lse, (total_len, 8)) # with T.block("root"): for lbx in T.thread_binding(16, thread="blockIdx.x"): for lby in T.thread_binding(1, thread="blockIdx.y"): for lty in T.thread_binding(4, thread="threadIdx.y"): for ltx in T.thread_binding(32, thread="threadIdx.x"): with T.block("attn"): bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) T.reads() T.writes() tile_id = T.alloc_buffer((1,), "int32", scope="local") batch_idx = T.alloc_buffer((1,), "int32", scope="local") batch_tiles = T.alloc_buffer((1,), "int32", scope="local") batch_rows = T.alloc_buffer((1,), "int32", scope="local") iterator = T.alloc_buffer((1,), "int32", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") Q_smem = T.alloc_buffer((16, 256), "float16", scope="shared") K_smem = T.alloc_buffer((16, 256), "float16", scope="shared") V_smem = T.alloc_buffer((16, 256), "float16", scope="shared") S_smem = T.alloc_buffer((16, 16), scope="shared") S_local = T.alloc_buffer((16, 16), scope="local") O_local = T.alloc_buffer((16, 256), scope="local") m_smem = T.alloc_buffer((16,), scope="shared") m_prev_smem = T.alloc_buffer((16,), scope="shared") d_smem = T.alloc_buffer((16,), scope="shared") m_new = T.alloc_buffer((1,), scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_new = T.alloc_buffer((1,), scope="local") tile_id[0] = bx batch_idx[0] = 0 batch_rows[0] = (q_indptr[1] - q_indptr[0]) * 8 batch_tiles[0] = (batch_rows[0] + 16 - 1) // 16 while T.tvm_thread_invariant(batch_idx[0] < batch_size): while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: tile_id[0] = tile_id[0] - batch_tiles[0] batch_idx[0] = batch_idx[0] + 1 if batch_idx[0] < batch_size: b_idx: T.int32 = batch_idx[0] batch_rows[0] = (q_indptr[b_idx + 1] - q_indptr[b_idx]) * 8 batch_tiles[0] = (batch_rows[0] + 16 - 1) // 16 if T.tvm_thread_invariant(batch_idx[0] < batch_size): b_idx: T.int32 = batch_idx[0] LH_start: T.int32 = tile_id[0] * 16 q_indptr_val: T.int32 = q_indptr[b_idx] cur_page_indptr_begin: T.int32 = page_indptr[b_idx] cur_page_indptr_end: T.int32 = page_indptr[b_idx + 1] kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[0, b_idx] - length_info[1, b_idx] + length_info[2, b_idx], 0) T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: m_smem[row] = T.float32(-50000.0) d_smem[row] = T.float32(1.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 8): with T.block("O_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) T.reads() T.writes(O_local[i, j]) O_local[i, j] = T.float32(0.0) T.tvm_storage_sync("shared") for li_lj_fused_0 in range(8): for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for li_lj_fused_3 in T.vectorized(4): with T.block("Q_load"): i = T.axis.spatial(16, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 256) j = T.axis.spatial(256, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = q_indptr_val + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: freq = T.float32() Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 128, q[cur_L, cur_H_qo, j + 128] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 128]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 256) / T.float32(256.0))}), q[cur_L, cur_H_qo, j]) else: Q_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for iterator_1 in range((kv_chunk_len[0] + 15) // 16): L_kv_start: T.int32 = iterator_1 * 16 for lz_ly_fused_0 in range(8): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("K_load"): i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 256) j = T.axis.spatial(256, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = T.if_then_else(cur_L < length_info[2, b_idx], cur_L, cur_L - length_info[2, b_idx] + length_info[1, b_idx]) page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 freq = T.float32() K_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 128, pages[page_no, 0, by, page_offset, j + 128] * T.float16(-1.0), pages[page_no, 0, by, page_offset, j - 128]))), where={freq: T.Cast("float32", k_rope_pos_offset[b_idx] + cur_L) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 256) / T.float32(256.0))}), pages[page_no, 0, by, page_offset, j]) else: K_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for lz_ly_fused_0 in range(8): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("V_load"): i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 256) j = T.axis.spatial(256, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = T.if_then_else(cur_L < length_info[2, b_idx], cur_L, cur_L - length_info[2, b_idx] + length_info[1, b_idx]) page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16] page_offset: T.int32 = seq_offset % 16 V_smem[i, j] = pages[page_no, 1, by, page_offset, j] else: V_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") with T.block(""): T.reads(Q_smem[0:16, 0:256], K_smem[0:16, 0:256]) T.writes(S_local[0:16, 0:16]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(1, 2): with T.block("S_gemm_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 + li_1_init) j = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 2 + lj_1_init) T.reads() T.writes(S_local[i, j]) S_local[i, j] = T.float32(0.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, li_1, lj_1, lk_1 in T.grid(32, 1, 2, 8): with T.block("S_gemm_update"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 + li_1) j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1) k = T.axis.reduce(256, lk_0 * 8 + lk_1) T.reads(S_local[i, j], Q_smem[i, k], K_smem[j, k]) T.writes(S_local[i, j]) S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k]) * T.Cast("float32", K_smem[j, k]) * attn_score_scaling_factor * T.float32(0.090168440055560212) T.tvm_storage_sync("shared") for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(1, 2): with T.block("S_store"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 + li_1) j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1) T.reads(S_local[i, j]) T.writes(S_smem[i, j]) S_smem[i, j] = S_local[i, j] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: with T.block("update1"): T.reads(m_smem[row], kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:16], d_smem[row], m_prev[i]) T.writes(m_prev[i], m_new[i], d_new[i]) m_prev[i] = m_smem[row] m_new[i] = m_smem[row] row_: T.int32 = (LH_start + row) // 8 for j in range(16): if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): m_new[i] = T.max(m_new[i], S_smem[row, j]) d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx with T.block("update"): T.reads(kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:16], m_new[i]) T.writes(S_smem[row, 0:16]) for j in range(16): if row < 16: row_: T.int32 = (LH_start + row) // 8 if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) else: S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: with T.block("update"): T.reads(d_new[i], S_smem[row, 0:16], m_new[i], m_prev[i]) T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) for j in range(16): d_new[i] = d_new[i] + S_smem[row, j] m_smem[row] = m_new[i] d_smem[row] = d_new[i] m_prev_smem[row] = m_prev[i] T.tvm_storage_sync("shared") with T.block(""): T.reads(m_prev_smem[0:16], m_smem[0:16], S_smem[0:16, 0:16], V_smem[0:16, 0:256]) T.writes(O_local[0:16, 0:256]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(4, 8): with T.block("O_gemm_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 32 * 4 + li_1_init) j = T.axis.spatial(256, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 32 * 8 + lj_1_init) T.reads() T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, lk_1, li_1, lj_1 in T.grid(2, 8, 4, 8): with T.block("O_gemm_update"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) k = T.axis.reduce(16, lk_0 * 8 + lk_1) T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k], V_smem[k, j]) T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] + S_smem[i, k] * T.Cast("float32", V_smem[k, j]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 8): with T.block("O_store"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) T.writes(output[q_indptr[b_idx] + (LH_start + i) // 8, by * 8 + (LH_start + i) % 8, j]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) for li_0 in range(1): for li_1 in T.thread_binding(4, thread="threadIdx.y"): for li_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("lse_store"): i = T.axis.spatial(16, li_0 * 128 + li_1 * 32 + li_2) T.where((li_0 * 4 + li_1) * 32 + li_2 < 16) T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) T.writes(lse[q_indptr[b_idx] + (LH_start + i) // 8, by * 8 + (LH_start + i) % 8]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) tile_id[0] = tile_id[0] + 16 @T.prim_func def gemma_q0f16_batch_prefill_ragged_kv(var_q: T.handle, var_q_indptr: T.handle, var_k: T.handle, var_v: T.handle, var_kv_indptr: T.handle, var_q_rope_position: T.handle, var_k_rope_pos_offset: T.handle, var_output: T.handle, var_lse: T.handle, causal: T.int32, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) qo_len = T.int32(is_size_var=True) q = T.match_buffer(var_q, (qo_len, 8, 256), "float16") batch_size = T.int32(is_size_var=True) q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) kv_len = T.int32(is_size_var=True) k = T.match_buffer(var_k, (kv_len, 1, 256), "float16") v = T.match_buffer(var_v, (kv_len, 1, 256), "float16") kv_indptr = T.match_buffer(var_kv_indptr, (batch_size + 1,), "int32", offset_factor=1) q_rope_position = T.match_buffer(var_q_rope_position, (qo_len,), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1) output = T.match_buffer(var_output, (qo_len, 8, 256), "float16") lse = T.match_buffer(var_lse, (qo_len, 8)) # with T.block("root"): for lbx in T.thread_binding(64, thread="blockIdx.x"): for lby in T.thread_binding(1, thread="blockIdx.y"): for lty in T.thread_binding(4, thread="threadIdx.y"): for ltx in T.thread_binding(32, thread="threadIdx.x"): with T.block("attn"): bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) T.reads() T.writes() tile_id = T.alloc_buffer((1,), "int32", scope="local") batch_idx = T.alloc_buffer((1,), "int32", scope="local") batch_tiles = T.alloc_buffer((1,), "int32", scope="local") batch_rows = T.alloc_buffer((1,), "int32", scope="local") iterator = T.alloc_buffer((1,), "int32", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") Q_smem = T.alloc_buffer((16, 256), "float16", scope="shared") K_smem = T.alloc_buffer((256, 16), "float16", scope="shared") V_smem = T.alloc_buffer((16, 256), "float16", scope="shared") S_smem = T.alloc_buffer((16, 16), scope="shared") S_local = T.alloc_buffer((16, 16), scope="local") O_local = T.alloc_buffer((16, 256), scope="local") m_smem = T.alloc_buffer((16,), scope="shared") m_prev_smem = T.alloc_buffer((16,), scope="shared") d_smem = T.alloc_buffer((16,), scope="shared") m_new = T.alloc_buffer((1,), scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_new = T.alloc_buffer((1,), scope="local") tile_id[0] = bx batch_idx[0] = 0 batch_rows[0] = (q_indptr[1] - q_indptr[0]) * 8 batch_tiles[0] = (batch_rows[0] + 16 - 1) // 16 while T.tvm_thread_invariant(batch_idx[0] < batch_size): while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: tile_id[0] = tile_id[0] - batch_tiles[0] batch_idx[0] = batch_idx[0] + 1 if batch_idx[0] < batch_size: b_idx: T.int32 = batch_idx[0] batch_rows[0] = (q_indptr[b_idx + 1] - q_indptr[b_idx]) * 8 batch_tiles[0] = (batch_rows[0] + 16 - 1) // 16 if T.tvm_thread_invariant(batch_idx[0] < batch_size): b_idx: T.int32 = batch_idx[0] q_indptr_val: T.int32 = q_indptr[b_idx] LH_start: T.int32 = tile_id[0] * 16 kv_chunk_len[0] = kv_indptr[b_idx + 1] - kv_indptr[b_idx] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: m_smem[row] = T.float32(-50000.0) d_smem[row] = T.float32(1.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1 in range(4): for lj_1_0 in T.unroll(1): for lj_1_1 in T.vectorized(8): with T.block("O_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1_0 * 8 + lj_1_1) T.reads() T.writes(O_local[i, j]) O_local[i, j] = T.float32(0.0) T.tvm_storage_sync("shared") for li_1_lj_0_1_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_1_lj_0_1_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_0, lj_0_0 in T.grid(2, 2): for lj_1 in T.vectorized(8): with T.block("Q_load"): i = T.axis.spatial(16, li_0 * 8 + (li_1_lj_0_1_fused_0 * 32 + li_1_lj_0_1_fused_1) // 16) j = T.axis.spatial(256, lj_0_0 * 128 + (li_1_lj_0_1_fused_0 * 32 + li_1_lj_0_1_fused_1) % 16 * 8 + lj_1) T.reads() T.writes() cur_L: T.int32 = q_indptr_val + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: freq = T.float32() Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 128, q[cur_L, cur_H_qo, j + 128] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 128]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 256) / T.float32(256.0))}), q[cur_L, cur_H_qo, j]) else: Q_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for iterator_1 in range((kv_chunk_len[0] + 15) // 16): L_kv_start: T.int32 = iterator_1 * 16 L_kv_base: T.int32 = kv_indptr[b_idx] for lz_1_ly_0_1_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for lz_1_ly_0_1_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lz_0, ly_0_0 in T.grid(2, 2): for ly_1 in T.vectorized(8): with T.block("K_load"): i = T.axis.spatial(16, lz_0 * 8 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) // 16) j = T.axis.spatial(256, ly_0_0 * 128 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) % 16 * 8 + ly_1) T.reads(kv_chunk_len[0], k_rope_pos_offset[b_idx], k[L_kv_base + L_kv_start + i, by, j - 128:j - 128 + 257]) T.writes(K_smem[j, i]) cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: freq = T.float32() K_smem[j, i] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", k[L_kv_base + cur_L, by, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 128, k[L_kv_base + cur_L, by, j + 128] * T.float16(-1.0), k[L_kv_base + cur_L, by, j - 128]))), where={freq: T.Cast("float32", k_rope_pos_offset[b_idx] + cur_L) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 256) / T.float32(256.0))}), k[L_kv_base + cur_L, by, j]) else: K_smem[j, i] = T.float16(0.0) T.tvm_storage_sync("shared") for lz_1_ly_0_1_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for lz_1_ly_0_1_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lz_0, ly_0_0 in T.grid(2, 2): for ly_1 in T.vectorized(8): with T.block("V_load"): i = T.axis.spatial(16, lz_0 * 8 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) // 16) j = T.axis.spatial(256, ly_0_0 * 128 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) % 16 * 8 + ly_1) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: V_smem[i, j] = v[L_kv_base + cur_L, by, j] else: V_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") with T.block(""): T.reads(Q_smem[0:16, 0:256], K_smem[0:256, 0:16]) T.writes(S_local[0:16, 0:16]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init in T.unroll(1): for lj_1_0_init in T.unroll(1): for lj_1_1_init in T.vectorized(2): with T.block("S_gemm_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 + li_1_init) j = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 2 + lj_1_0_init * 2 + lj_1_1_init) T.reads() T.writes(S_local[i, j]) S_local[i, j] = T.float32(0.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0 in range(16): for li_1 in T.unroll(1): for lj_1_0 in T.unroll(1): for lj_1_1 in T.vectorized(2): for lk_1 in range(16): with T.block("S_gemm_update"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 + li_1) j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1_0 * 2 + lj_1_1) k_1 = T.axis.reduce(256, lk_0 * 16 + lk_1) T.reads(S_local[i, j], Q_smem[i, k_1], K_smem[k_1, j]) T.writes(S_local[i, j]) S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k_1]) * T.Cast("float32", K_smem[k_1, j]) * attn_score_scaling_factor * T.float32(0.090168440055560212) T.tvm_storage_sync("shared") for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1 in range(1): for lj_1_0 in T.unroll(1): for lj_1_1 in T.vectorized(2): with T.block("S_store"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 + li_1) j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1_0 * 2 + lj_1_1) T.reads(S_local[i, j]) T.writes(S_smem[i, j]) S_smem[i, j] = S_local[i, j] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: with T.block("update1"): T.reads(m_smem[row], kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:16], d_smem[row], m_prev[i]) T.writes(m_prev[i], m_new[i], d_new[i]) m_prev[i] = m_smem[row] m_new[i] = m_smem[row] row_: T.int32 = (LH_start + row) // 8 for j in range(16): if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): m_new[i] = T.max(m_new[i], S_smem[row, j]) d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx with T.block("update"): T.reads(kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:16], m_new[i]) T.writes(S_smem[row, 0:16]) for j in range(16): if row < 16: row_: T.int32 = (LH_start + row) // 8 if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]): S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) else: S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: with T.block("update"): T.reads(d_new[i], S_smem[row, 0:16], m_new[i], m_prev[i]) T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) for j in range(16): d_new[i] = d_new[i] + S_smem[row, j] m_smem[row] = m_new[i] d_smem[row] = d_new[i] m_prev_smem[row] = m_prev[i] T.tvm_storage_sync("shared") with T.block(""): T.reads(m_prev_smem[0:16], m_smem[0:16], S_smem[0:16, 0:16], V_smem[0:16, 0:256]) T.writes(O_local[0:16, 0:256]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init in T.unroll(4): for lj_1_0_init in T.unroll(1): for lj_1_1_init in T.vectorized(8): with T.block("O_gemm_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 32 * 4 + li_1_init) j = T.axis.spatial(256, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 32 * 8 + lj_1_0_init * 8 + lj_1_1_init) T.reads() T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, lk_1 in T.grid(1, 16): for li_1 in T.unroll(4): for lj_1_0 in T.unroll(1): for lj_1_1 in T.vectorized(8): with T.block("O_gemm_update"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1_0 * 8 + lj_1_1) k_1 = T.axis.reduce(16, lk_0 * 16 + lk_1) T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k_1], V_smem[k_1, j]) T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] + S_smem[i, k_1] * T.Cast("float32", V_smem[k_1, j]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1 in range(4): for lj_1_0 in T.unroll(1): for lj_1_1 in T.vectorized(8): with T.block("O_store"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1_0 * 8 + lj_1_1) T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) T.writes(output[q_indptr[b_idx] + (LH_start + i) // 8, by * 8 + (LH_start + i) % 8, j]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) for li_0 in range(1): for li_1 in T.thread_binding(4, thread="threadIdx.y"): for li_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("lse_store"): i = T.axis.spatial(16, li_0 * 128 + li_1 * 32 + li_2) T.where((li_0 * 4 + li_1) * 32 + li_2 < 16) T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) T.writes(lse[q_indptr[b_idx] + (LH_start + i) // 8, by * 8 + (LH_start + i) % 8]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) tile_id[0] = tile_id[0] + 64 @T.prim_func def gemma_q0f16_batch_tree_attn(var_q: T.handle, var_q_indptr: T.handle, var_k: T.handle, var_v: T.handle, var_kv_indptr: T.handle, var_q_rope_position: T.handle, var_mn_indptr: T.handle, var_mask: T.handle, var_output: T.handle, var_lse: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32, batch_size: T.int32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) qo_len = T.int32(is_size_var=True) q = T.match_buffer(var_q, (qo_len, 8, 256), "float16") q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) kv_len = T.int32(is_size_var=True) k = T.match_buffer(var_k, (kv_len, 1, 256), "float16") v = T.match_buffer(var_v, (kv_len, 1, 256), "float16") kv_indptr = T.match_buffer(var_kv_indptr, (batch_size + 1,), "int32", offset_factor=1) q_rope_position = T.match_buffer(var_q_rope_position, (qo_len,), "int32", offset_factor=1) mn_indptr = T.match_buffer(var_mn_indptr, (batch_size + 1,), "int32", offset_factor=1) tree_size = T.int32(is_size_var=True) mask = T.match_buffer(var_mask, (tree_size, 2), "int32", offset_factor=1) output = T.match_buffer(var_output, (qo_len, 8, 256), "float16") lse = T.match_buffer(var_lse, (qo_len, 8)) # with T.block("root"): for lbx in T.thread_binding(16, thread="blockIdx.x"): for lby in T.thread_binding(1, thread="blockIdx.y"): for lty in T.thread_binding(4, thread="threadIdx.y"): for ltx in T.thread_binding(32, thread="threadIdx.x"): with T.block("attn"): bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) T.reads() T.writes() tile_id = T.alloc_buffer((1,), "int32", scope="local") batch_idx = T.alloc_buffer((1,), "int32", scope="local") batch_tiles = T.alloc_buffer((1,), "int32", scope="local") batch_rows = T.alloc_buffer((1,), "int32", scope="local") iterator = T.alloc_buffer((1,), "int32", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") Q_smem = T.alloc_buffer((16, 256), "float16", scope="shared") K_smem = T.alloc_buffer((32, 256), "float16", scope="shared") V_smem = T.alloc_buffer((32, 256), "float16", scope="shared") S_smem = T.alloc_buffer((16, 32), scope="shared") S_local = T.alloc_buffer((16, 32), scope="local") O_local = T.alloc_buffer((16, 256), scope="local") m_smem = T.alloc_buffer((16,), scope="shared") m_prev_smem = T.alloc_buffer((16,), scope="shared") d_smem = T.alloc_buffer((16,), scope="shared") m_new = T.alloc_buffer((1,), scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_new = T.alloc_buffer((1,), scope="local") tile_id[0] = bx batch_idx[0] = 0 batch_rows[0] = (q_indptr[1] - q_indptr[0]) * 8 batch_tiles[0] = (batch_rows[0] + 16 - 1) // 16 while T.tvm_thread_invariant(batch_idx[0] < batch_size): while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: tile_id[0] = tile_id[0] - batch_tiles[0] batch_idx[0] = batch_idx[0] + 1 if batch_idx[0] < batch_size: b_idx: T.int32 = batch_idx[0] batch_rows[0] = (q_indptr[b_idx + 1] - q_indptr[b_idx]) * 8 batch_tiles[0] = (batch_rows[0] + 16 - 1) // 16 if T.tvm_thread_invariant(batch_idx[0] < batch_size): b_idx: T.int32 = batch_idx[0] LH_start: T.int32 = tile_id[0] * 16 q_indptr_val: T.int32 = q_indptr[b_idx] kv_chunk_len[0] = kv_indptr[b_idx + 1] - kv_indptr[b_idx] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: m_smem[row] = T.float32(-50000.0) d_smem[row] = T.float32(1.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 8): with T.block("O_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) T.reads() T.writes(O_local[i, j]) O_local[i, j] = T.float32(0.0) T.tvm_storage_sync("shared") for li_lj_fused_0 in range(8): for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for li_lj_fused_3 in T.vectorized(4): with T.block("Q_load"): i = T.axis.spatial(16, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 256) j = T.axis.spatial(256, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = q_indptr_val + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: freq = T.float32() Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 128, q[cur_L, cur_H_qo, j + 128] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 128]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 256) / T.float32(256.0))}), q[cur_L, cur_H_qo, j]) else: Q_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for iterator_1 in range((kv_chunk_len[0] + 31) // 32): L_kv_start: T.int32 = iterator_1 * 32 L_kv_base: T.int32 = kv_indptr[b_idx] for lz_ly_fused_0 in range(16): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("KV_load"): i = T.axis.spatial(32, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 256) j = T.axis.spatial(256, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = L_kv_base + L_kv_start + i if L_kv_start + i < kv_chunk_len[0]: freq = T.float32() K_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", k[cur_L, by, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 128, k[cur_L, by, j + 128] * T.float16(-1.0), k[cur_L, by, j - 128]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 256) / T.float32(256.0))}), k[cur_L, by, j]) V_smem[i, j] = v[cur_L, by, j] else: K_smem[i, j] = T.float16(0.0) V_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") with T.block(""): T.reads(Q_smem[0:16, 0:256], K_smem[0:32, 0:256]) T.writes(S_local[0:16, 0:32]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(2, 2): with T.block("S_gemm_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 2 + li_1_init) j = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 2 + lj_1_init) T.reads() T.writes(S_local[i, j]) S_local[i, j] = T.float32(0.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, li_1, lj_1, lk_1 in T.grid(32, 2, 2, 8): with T.block("S_gemm_update"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 2 + li_1) j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 2 + lj_1) k_1 = T.axis.reduce(256, lk_0 * 8 + lk_1) T.reads(S_local[i, j], Q_smem[i, k_1], K_smem[j, k_1]) T.writes(S_local[i, j]) S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k_1]) * T.Cast("float32", K_smem[j, k_1]) * attn_score_scaling_factor * T.float32(0.090168440055560212) T.tvm_storage_sync("shared") for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(2, 2): with T.block("S_store"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 2 + li_1) j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 2 + lj_1) T.reads(S_local[i, j]) T.writes(S_smem[i, j]) S_smem[i, j] = S_local[i, j] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: with T.block("update1"): T.reads(m_smem[row], kv_chunk_len[0], mn_indptr[b_idx:b_idx + 2], mask[T.min((LH_start + row) // 8 + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]):T.min((LH_start + row) // 8 + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max((LH_start + row) // 8 + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min((LH_start + row) // 8 + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:32], d_smem[row], m_prev[i]) T.writes(m_prev[i], m_new[i], d_new[i]) m_prev[i] = m_smem[row] m_new[i] = m_smem[row] row_: T.int32 = (LH_start + row) // 8 for j in range(32): if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) or mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 0] and mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 1]): m_new[i] = T.max(m_new[i], S_smem[row, j]) d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx with T.block("update"): T.reads(kv_chunk_len[0], mn_indptr[b_idx:b_idx + 2], mask[T.min((LH_start + row) // 8 + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]):T.min((LH_start + row) // 8 + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max((LH_start + row) // 8 + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min((LH_start + row) // 8 + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:32], m_new[i]) T.writes(S_smem[row, 0:32]) for j in range(32): if row < 16: row_: T.int32 = (LH_start + row) // 8 if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) or mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 0] and mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 1]): S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) else: S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: with T.block("update"): T.reads(d_new[i], S_smem[row, 0:32], m_new[i], m_prev[i]) T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) for j in range(32): d_new[i] = d_new[i] + S_smem[row, j] m_smem[row] = m_new[i] d_smem[row] = d_new[i] m_prev_smem[row] = m_prev[i] T.tvm_storage_sync("shared") with T.block(""): T.reads(m_prev_smem[0:16], m_smem[0:16], S_smem[0:16, 0:32], V_smem[0:32, 0:256]) T.writes(O_local[0:16, 0:256]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(4, 8): with T.block("O_gemm_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 32 * 4 + li_1_init) j = T.axis.spatial(256, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 32 * 8 + lj_1_init) T.reads() T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, lk_1, li_1, lj_1 in T.grid(4, 8, 4, 8): with T.block("O_gemm_update"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) k_1 = T.axis.reduce(32, lk_0 * 8 + lk_1) T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k_1], V_smem[k_1, j]) T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] + S_smem[i, k_1] * T.Cast("float32", V_smem[k_1, j]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 8): with T.block("O_store"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) T.writes(output[q_indptr[b_idx] + (LH_start + i) // 8, by * 8 + (LH_start + i) % 8, j]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) for li_0 in range(1): for li_1 in T.thread_binding(4, thread="threadIdx.y"): for li_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("lse_store"): i = T.axis.spatial(16, li_0 * 128 + li_1 * 32 + li_2) T.where((li_0 * 4 + li_1) * 32 + li_2 < 16) T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) T.writes(lse[q_indptr[b_idx] + (LH_start + i) // 8, by * 8 + (LH_start + i) % 8]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) tile_id[0] = tile_id[0] + 16 @T.prim_func def gemma_q0f16_chunk_lse(var_A: T.handle, var_temperature: T.handle, var_chunked_sum: T.handle, var_chunked_max: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True) A = T.match_buffer(var_A, (batch_size, vocab_size)) temperature = T.match_buffer(var_temperature, (batch_size,)) num_chunks = T.int32(is_size_var=True) chunked_sum = T.match_buffer(var_chunked_sum, (batch_size, num_chunks)) chunked_max = T.match_buffer(var_chunked_max, (batch_size, num_chunks)) # with T.block("root"): temp_max_shared = T.alloc_buffer((batch_size, num_chunks), scope="shared") temp_sum_shared = T.alloc_buffer((batch_size, num_chunks), scope="shared") for ax0_ax1_fused in T.thread_binding(batch_size * num_chunks, thread="blockIdx.x"): for ax0, ax1 in T.grid(1, 1): for ax2_fused_1 in T.thread_binding(256, thread="threadIdx.x"): for ax2_fused_0 in T.serial(16, annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("max"): v0 = T.axis.spatial(batch_size, ax0_ax1_fused % (num_chunks * batch_size) // num_chunks + ax0) v1 = T.axis.spatial(num_chunks, ax0_ax1_fused % num_chunks + ax1) v2 = T.axis.reduce(4096, ax2_fused_0 * 256 + ax2_fused_1) T.reads(temperature[v0], A[v0, v1 * 4096 + v2]) T.writes(temp_max_shared[v0, v1]) with T.init(): temp_max_shared[v0, v1] = T.float32(-340282346638528859811704183484516925440.0) temp_max_shared[v0, v1] = T.max(temp_max_shared[v0, v1], T.if_then_else(v1 * 4096 + v2 < vocab_size, T.if_then_else(temperature[v0] > T.float32(1.0000000000000001e-05), A[v0, v1 * 4096 + v2] / temperature[v0], A[v0, v1 * 4096 + v2]), T.float32(-340282346638528859811704183484516925440.0))) for ax0, ax1 in T.grid(1, 1): for ax2_fused_1 in T.thread_binding(256, thread="threadIdx.x"): for ax2_fused_0 in T.serial(16, annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("sum_exp"): v0 = T.axis.spatial(batch_size, ax0_ax1_fused % (num_chunks * batch_size) // num_chunks + ax0) v1 = T.axis.spatial(num_chunks, ax0_ax1_fused % num_chunks + ax1) v2 = T.axis.reduce(4096, ax2_fused_0 * 256 + ax2_fused_1) T.reads(temperature[v0], A[v0, v1 * 4096 + v2], temp_max_shared[v0, v1]) T.writes(temp_sum_shared[v0, v1]) with T.init(): temp_sum_shared[v0, v1] = T.float32(0.0) temp_sum_shared[v0, v1] = temp_sum_shared[v0, v1] + T.if_then_else(v1 * 4096 + v2 < vocab_size, T.Select(temperature[v0] > T.float32(1.0000000000000001e-05), T.exp(T.if_then_else(v1 * 4096 + v2 < vocab_size, T.if_then_else(temperature[v0] > T.float32(1.0000000000000001e-05), A[v0, v1 * 4096 + v2] / temperature[v0], A[v0, v1 * 4096 + v2]), T.float32(-340282346638528859811704183484516925440.0)) - temp_max_shared[v0, v1]), T.Cast("float32", T.if_then_else(v1 * 4096 + v2 < vocab_size, T.if_then_else(temperature[v0] > T.float32(1.0000000000000001e-05), A[v0, v1 * 4096 + v2] / temperature[v0], A[v0, v1 * 4096 + v2]), T.float32(-340282346638528859811704183484516925440.0)) == temp_max_shared[v0, v1])), T.float32(0.0)) for ax2_1 in T.thread_binding(256, thread="threadIdx.x"): for ax2_0 in T.serial(1, annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("log"): v0 = T.axis.spatial(batch_size, ax0_ax1_fused % (num_chunks * batch_size) // num_chunks) v1 = T.axis.spatial(num_chunks, ax0_ax1_fused % num_chunks) v2 = T.axis.spatial(1, ax2_0 * 256 + ax2_1) T.where(ax2_0 * 256 + ax2_1 < 1) T.reads(temperature[v0], temp_sum_shared[v0, v1], temp_max_shared[v0, v1]) T.writes(chunked_sum[v0, v1], chunked_max[v0, v1]) chunked_sum[v0, v1] = T.Select(temperature[v0] > T.float32(1.0000000000000001e-05), T.log(temp_sum_shared[v0, v1]), temp_sum_shared[v0, v1]) chunked_max[v0, v1] = temp_max_shared[v0, v1] @T.prim_func def gemma_q0f16_compact_kv_copy(var_pages: T.handle, var_copy_length_indptr: T.handle, var_copy_src_dst_pos: T.handle, batch_size: T.int32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) num_pages = T.int32() pages = T.match_buffer(var_pages, (num_pages, 2, 1, 16, 256), "float16", offset_factor=1) copy_length_indptr = T.match_buffer(var_copy_length_indptr, (batch_size + 1,), "int32", offset_factor=1) total_copy_length = T.int32() copy_src_dst_pos = T.match_buffer(var_copy_src_dst_pos, (2, total_copy_length), "int32", offset_factor=1) with T.block("root"): T.reads() T.writes() for bhd_o in T.thread_binding(batch_size, thread="blockIdx.x"): for bhd_i in T.thread_binding(256, thread="threadIdx.x"): b: T.int32 = (bhd_o * 256 + bhd_i) // 256 h: T.int32 = 0 d: T.int32 = (bhd_o * 256 + bhd_i) % 256 if bhd_o * 256 + bhd_i < batch_size * 256: for i in range(copy_length_indptr[b + 1] - copy_length_indptr[b]): src_pos: T.int32 = copy_src_dst_pos[0, copy_length_indptr[b] + i] dst_pos: T.int32 = copy_src_dst_pos[1, copy_length_indptr[b] + i] pages[dst_pos // 16, 0, h, dst_pos % 16, d] = pages[src_pos // 16, 0, h, src_pos % 16, d] pages[dst_pos // 16, 1, h, dst_pos % 16, d] = pages[src_pos // 16, 1, h, src_pos % 16, d] @T.prim_func def gemma_q0f16_copy_single_page(var_pages: T.handle, src_page_id: T.int32, tgt_page_id: T.int32, copy_length: T.int32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) num_pages, page_size = T.int32(), T.int32() pages = T.match_buffer(var_pages, (num_pages, 2, 1, page_size, 256), "float16", offset_factor=1) # with T.block("root"): for b in T.thread_binding(copy_length, thread="blockIdx.x"): for t in T.thread_binding(256, thread="threadIdx.x"): with T.block("copy"): vh = T.axis.spatial(1, (b * 256 + t) // (copy_length * 256)) vp = T.axis.spatial(copy_length, (b * 256 + t) % (copy_length * 256) // 256) vd = T.axis.spatial(256, (b * 256 + t) % 256) T.where(b * 256 + t < copy_length * 256) T.reads(pages[src_page_id, 0:2, vh, vp, vd]) T.writes(pages[tgt_page_id, 0:2, vh, vp, vd]) pages[tgt_page_id, 0, vh, vp, vd] = pages[src_page_id, 0, vh, vp, vd] pages[tgt_page_id, 1, vh, vp, vd] = pages[src_page_id, 1, vh, vp, vd] @T.prim_func def gemma_q0f16_fuse_add_norm_decode(pA: T.handle, pB: T.handle, C: T.Buffer((2048,), "float16"), pO: T.handle, pAdd: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() A = T.match_buffer(pA, (batch_size, 1, 2048), "float16") B = T.match_buffer(pB, (batch_size, 1, 2048), "float16") O = T.match_buffer(pO, (batch_size, 1, 2048), "float16") add = T.match_buffer(pAdd, (batch_size, 1, 2048), "float16") # with T.block("root"): add_local = T.alloc_buffer((8,), "float16", scope="local") sum_shared = T.alloc_buffer((batch_size, 1), scope="shared") sum_local = T.alloc_buffer((256, batch_size, 1), scope="local") for v_bx in T.thread_binding(batch_size, thread="blockIdx.x"): for v_tx in T.thread_binding(256, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 256, "pragma_unroll_explicit": 1}): for i in range(8): with T.block("T_add"): bx = T.axis.spatial(batch_size, v_bx) h = T.axis.spatial(2048, i * 256 + v_tx) T.reads(A[bx, 0, h], B[bx, 0, h]) T.writes(add_local[h // 256]) add_local[h // 256] = A[bx, 0, h] + B[bx, 0, h] with T.block("T_write_back"): bx = T.axis.spatial(batch_size, v_bx) v_ax1 = T.axis.spatial(1, 0) h = T.axis.spatial(2048, i * 256 + v_tx) T.reads(add_local[h // 256]) T.writes(add[bx, v_ax1, h]) add[bx, v_ax1, h] = add_local[h // 256] with T.block("T_multiply_red_rf_init"): tx, bx = T.axis.remap("SS", [v_tx, v_bx]) T.reads() T.writes(sum_local[tx, bx, 0]) sum_local[tx, bx, 0] = T.float32(0.0) for v_i, _j in T.grid(8, 1): with T.block("T_multiply_red_rf_update"): tx, bx, i = T.axis.remap("SSR", [v_tx, v_bx, v_i]) T.reads(sum_local[tx, bx, 0], add_local[i]) T.writes(sum_local[tx, bx, 0]) sum_local[tx, bx, 0] = sum_local[tx, bx, 0] + T.Cast("float32", add_local[i]) * T.Cast("float32", add_local[i]) for _j in range(1): for v_tx_2 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_multiply_red"): tx, bx = T.axis.remap("RS", [v_tx_2, v_bx]) T.reads(sum_local[tx, bx, 0]) T.writes(sum_shared[bx, 0]) with T.init(): sum_shared[bx, 0] = T.float32(0.0) sum_shared[bx, 0] = sum_shared[bx, 0] + sum_local[tx, bx, 0] for i in range(8): for v_tx_2 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_cast_2"): bx = T.axis.spatial(batch_size, v_bx) h = T.axis.spatial(2048, i * 256 + v_tx_2) T.reads(sum_shared[bx, 0], add_local[h // 256], C[h]) T.writes(O[bx, 0, h]) O[bx, 0, h] = T.Cast("float16", T.rsqrt(sum_shared[bx, 0] * T.float32(0.00048828125) + T.float32(9.9999999999999995e-07)) * T.Cast("float32", add_local[h // 256]) * T.Cast("float32", C[h])) @T.prim_func def gemma_q0f16_fuse_add_norm_prefill(pA: T.handle, pB: T.handle, C: T.Buffer((2048,), "float16"), pO: T.handle, pAdd: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() A = T.match_buffer(pA, (1, seq_len, 2048), "float16") B = T.match_buffer(pB, (1, seq_len, 2048), "float16") O = T.match_buffer(pO, (1, seq_len, 2048), "float16") add = T.match_buffer(pAdd, (1, seq_len, 2048), "float16") # with T.block("root"): add_local = T.alloc_buffer((8,), "float16", scope="local") sum_shared = T.alloc_buffer((1, seq_len), scope="shared") sum_local = T.alloc_buffer((256, 1, seq_len), scope="local") for v_bx in T.thread_binding(seq_len, thread="blockIdx.x"): for v_tx in T.thread_binding(256, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 256, "pragma_unroll_explicit": 1}): for v_i in range(8): with T.block("T_add"): bx = T.axis.spatial(seq_len, v_bx) h = T.axis.spatial(2048, v_i * 256 + v_tx) T.reads(A[0, bx, h], B[0, bx, h]) T.writes(add_local[h // 256]) add_local[h // 256] = A[0, bx, h] + B[0, bx, h] with T.block("T_write_back"): bx = T.axis.spatial(seq_len, v_bx) h = T.axis.spatial(2048, v_i * 256 + v_tx) T.reads(add_local[h // 256]) T.writes(add[0, bx, h]) add[0, bx, h] = add_local[h // 256] with T.block("T_multiply_red_rf_init"): tx, bx = T.axis.remap("SS", [v_tx, v_bx]) T.reads() T.writes(sum_local[tx, 0, bx]) sum_local[tx, 0, bx] = T.float32(0.0) for v_i, _j in T.grid(8, 1): with T.block("T_multiply_red_rf_update"): tx, bx, i = T.axis.remap("SSR", [v_tx, v_bx, v_i]) T.reads(sum_local[tx, 0, bx], add_local[i]) T.writes(sum_local[tx, 0, bx]) sum_local[tx, 0, bx] = sum_local[tx, 0, bx] + T.Cast("float32", add_local[i]) * T.Cast("float32", add_local[i]) for _j in range(1): for v_tx_2 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_multiply_red"): tx, bx = T.axis.remap("RS", [v_tx_2, v_bx]) T.reads(sum_local[tx, 0, bx]) T.writes(sum_shared[0, bx]) with T.init(): sum_shared[0, bx] = T.float32(0.0) sum_shared[0, bx] = sum_shared[0, bx] + sum_local[tx, 0, bx] for v_i in range(8): for v_tx_2 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_cast_2"): bx = T.axis.spatial(seq_len, v_bx) v1 = T.axis.spatial(2048, v_i * 256 + v_tx_2) T.reads(sum_shared[0, bx], add_local[v1 // 256], C[v1]) T.writes(O[0, bx, v1]) O[0, bx, v1] = T.Cast("float16", T.rsqrt(sum_shared[0, bx] * T.float32(0.00048828125) + T.float32(9.9999999999999995e-07)) * T.Cast("float32", add_local[v1 // 256]) * T.Cast("float32", C[v1])) @T.prim_func def gemma_q0f16_fused_reshape10_reshape11(lv19: T.Buffer((1, 8, 256), "float16"), T_reshape_intermediate_1: T.Buffer((1, 1, 2048), "float16")): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) # with T.block("root"): for ax0_fused_0 in T.thread_binding(8, thread="blockIdx.x"): for ax0_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_reshape_1"): v0 = T.axis.spatial(2048, ax0_fused_0 * 256 + ax0_fused_1) T.reads(lv19[0, v0 // 256, v0 % 256]) T.writes(T_reshape_intermediate_1[0, 0, v0]) T_reshape_intermediate_1[0, 0, v0] = lv19[0, v0 // 256, v0 % 256] @T.prim_func def gemma_q0f16_fused_reshape8_reshape9(lv219: T.Buffer((1, 1, 2560), "float16"), T_reshape_intermediate_1: T.Buffer((1, 10, 256), "float16")): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) # with T.block("root"): for ax0_ax1_fused_0 in T.thread_binding(10, thread="blockIdx.x"): for ax0_ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_reshape_1"): v0 = T.axis.spatial(10, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) // 256) v1 = T.axis.spatial(256, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) % 256) T.reads(lv219[0, 0, v0 * 256 + v1]) T.writes(T_reshape_intermediate_1[0, v0, v1]) T_reshape_intermediate_1[0, v0, v1] = lv219[0, 0, v0 * 256 + v1] @T.prim_func def gemma_q0f16_fused_rope(var_qkv: T.handle, var_position_map: T.handle, var_q: T.handle, var_k: T.handle, var_v: T.handle, apply_rope: T.int32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() qkv = T.match_buffer(var_qkv, (seq_len, 10, 256), "float16") position_map = T.match_buffer(var_position_map, (seq_len,), "int32", offset_factor=1) q = T.match_buffer(var_q, (seq_len, 8, 256), "float16") k = T.match_buffer(var_k, (seq_len, 1, 256), "float16") v = T.match_buffer(var_v, (seq_len, 1, 256), "float16") # with T.block("root"): for ax0_ax1_ax2_fused_0 in T.thread_binding(seq_len * 10, thread="blockIdx.x"): for ax0_ax1_ax2_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("llama_fused_rope"): v0 = T.axis.spatial(seq_len, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) // 2560) v1 = T.axis.spatial(10, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 2560 // 256) v2 = T.axis.spatial(256, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 256) T.reads(position_map[v0], qkv[v0, v1, v2 + -128:v2 + -128 + 257]) T.writes(q[v0, v1, v2], k[v0, v1 + -8, v2], v[v0, v1 + -9, v2]) if v1 < 8: freq = T.float32() q[v0, v1, v2] = T.if_then_else(apply_rope > 0 and v2 < 256, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", qkv[v0, v1, v2]) + T.sin(freq) * T.Cast("float32", T.if_then_else(v2 < 128, qkv[v0, v1, v2 + 128] * T.float16(-1.0), qkv[v0, v1, v2 + -128]))), where={freq: T.Cast("float32", position_map[v0]) / T.pow(T.float32(10000.0), T.Cast("float32", v2 * 2 % 256) / T.float32(256.0))}), qkv[v0, v1, v2]) else: if v1 < 9: freq = T.float32() k[v0, v1 + -8, v2] = T.if_then_else(apply_rope > 0 and v2 < 256, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", qkv[v0, v1, v2]) + T.sin(freq) * T.Cast("float32", T.if_then_else(v2 < 128, qkv[v0, v1, v2 + 128] * T.float16(-1.0), qkv[v0, v1, v2 + -128]))), where={freq: T.Cast("float32", position_map[v0]) / T.pow(T.float32(10000.0), T.Cast("float32", v2 * 2 % 256) / T.float32(256.0))}), qkv[v0, v1, v2]) else: v[v0, v1 + -9, v2] = qkv[v0, v1, v2] @T.prim_func def gemma_q0f16_fused_split1_gelu_tanh1_multiply3(p_lv75: T.handle, p_output0: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() lv75 = T.match_buffer(p_lv75, (1, seq_len, 32768), "float16") T_multiply_intermediate_1 = T.match_buffer(p_output0, (1, seq_len, 16384), "float16") # with T.block("root"): for ax0_ax1_fused_0 in T.thread_binding(seq_len * 64, thread="blockIdx.x"): for ax0_ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_multiply_6"): v0 = T.axis.spatial(seq_len, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) // 16384) v1 = T.axis.spatial(16384, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) % 16384) T.reads(lv75[0, v0, v1:v1 + 16385]) T.writes(T_multiply_intermediate_1[0, v0, v1]) T_multiply_intermediate_1[0, v0, v1] = T.float16(0.5) * lv75[0, v0, v1] * (T.float16(1.0) + T.tanh(T.float16(0.79788456080286541) * lv75[0, v0, v1] * (T.float16(1.0) + T.float16(0.044714999999999998) * lv75[0, v0, v1] * lv75[0, v0, v1]))) * lv75[0, v0, v1 + 16384] @T.prim_func def gemma_q0f16_fused_split2_gelu_tanh2_multiply5(lv221: T.Buffer((1, 1, 32768), "float16"), T_multiply_intermediate_1: T.Buffer((1, 1, 16384), "float16")): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) # with T.block("root"): for ax0_fused_0 in T.thread_binding(64, thread="blockIdx.x"): for ax0_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_multiply_6"): v0 = T.axis.spatial(16384, ax0_fused_0 * 256 + ax0_fused_1) T.reads(lv221[0, 0, v0:v0 + 16385]) T.writes(T_multiply_intermediate_1[0, 0, v0]) T_multiply_intermediate_1[0, 0, v0] = T.float16(0.5) * lv221[0, 0, v0] * (T.float16(1.0) + T.tanh(T.float16(0.79788456080286541) * lv221[0, 0, v0] * (T.float16(1.0) + T.float16(0.044714999999999998) * lv221[0, 0, v0] * lv221[0, 0, v0]))) * lv221[0, 0, v0 + 16384] @T.prim_func def gemma_q0f16_fused_split_gelu_tanh_multiply1(p_lv2: T.handle, p_output0: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() lv2 = T.match_buffer(p_lv2, (batch_size, 1, 32768), "float16") T_multiply_intermediate_1 = T.match_buffer(p_output0, (batch_size, 1, 16384), "float16") # with T.block("root"): for ax0_ax1_fused_0 in T.thread_binding(batch_size * 64, thread="blockIdx.x"): for ax0_ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_multiply_6"): v0 = T.axis.spatial(batch_size, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) // 16384) v1 = T.axis.spatial(16384, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) % 16384) T.reads(lv2[v0, 0, v1:v1 + 16385]) T.writes(T_multiply_intermediate_1[v0, 0, v1]) T_multiply_intermediate_1[v0, 0, v1] = T.float16(0.5) * lv2[v0, 0, v1] * (T.float16(1.0) + T.tanh(T.float16(0.79788456080286541) * lv2[v0, 0, v1] * (T.float16(1.0) + T.float16(0.044714999999999998) * lv2[v0, 0, v1] * lv2[v0, 0, v1]))) * lv2[v0, 0, v1 + 16384] @T.prim_func def gemma_q0f16_gather_probs(var_src: T.handle, var_indices: T.handle, var_dst: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) m, n = T.int32(is_size_var=True), T.int32(is_size_var=True) src = T.match_buffer(var_src, (m, n)) batch_size = T.int32(is_size_var=True) indices = T.match_buffer(var_indices, (batch_size,), "int32") dst = T.match_buffer(var_dst, (batch_size, n)) # with T.block("root"): for ax0_ax1_fused_0 in T.thread_binding((batch_size * n + 255) // 256, thread="blockIdx.x"): for ax0_ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("gather_2d"): v0 = T.axis.spatial(batch_size, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) % (n * batch_size) // n) v1 = T.axis.spatial(n, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) % n) T.where(ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1 < batch_size * n) T.reads(src[indices[v0], v1], indices[v0]) T.writes(dst[v0, v1]) dst[v0, v1] = src[indices[v0], v1] @T.prim_func def gemma_q0f16_index(var_rms_norm36: T.handle, index: T.Buffer((1, 1, 2048), "float16")): T.func_attr({"target": T.target({"keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() rms_norm36 = T.match_buffer(var_rms_norm36, (1, seq_len, 2048), "float16") # with T.block("root"): for ax0_fused_0 in T.thread_binding(8, thread="blockIdx.x"): for ax0_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("index"): v0 = T.axis.spatial(2048, ax0_fused_0 * 256 + ax0_fused_1) T.reads(rms_norm36[0, seq_len - 1, v0]) T.writes(index[0, 0, v0]) index[0, 0, v0] = rms_norm36[0, seq_len - 1, v0] @T.prim_func def gemma_q0f16_merge_state_inplace(v: T.handle, s: T.handle, v_other: T.handle, s_other: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) N, H, D = T.int32(is_size_var=True), T.int32(is_size_var=True), T.int32(is_size_var=True) V = T.match_buffer(v, (N, H, D), "float16") S = T.match_buffer(s, (N, H)) V_other = T.match_buffer(v_other, (N, H, D), "float16") S_other = T.match_buffer(s_other, (N, H)) # with T.block("root"): for bx in T.thread_binding(N, thread="blockIdx.x"): for by in T.thread_binding(2, thread="blockIdx.y"): for ty in T.thread_binding(4, thread="threadIdx.y"): for tx in T.thread_binding(64, thread="threadIdx.x"): with T.block("merge"): T.reads(S[bx, ty + by * 4], S_other[bx, ty + by * 4], V[bx, ty + by * 4, tx * 4:tx * 4 + 4], V_other[bx, ty + by * 4, tx * 4:tx * 4 + 4]) T.writes(V[bx, ty + by * 4, tx * 4:tx * 4 + 4], S[bx, ty + by * 4]) s_val = T.alloc_buffer((1,), scope="local") s_other_val = T.alloc_buffer((1,), scope="local") s_max = T.alloc_buffer((1,), scope="local") scale = T.alloc_buffer((1,), scope="local") other_scale = T.alloc_buffer((1,), scope="local") v_vec = T.alloc_buffer((4,), "float16", scope="local") v_other_vec = T.alloc_buffer((4,), "float16", scope="local") s_val[0] = S[bx, ty + by * 4] s_other_val[0] = S_other[bx, ty + by * 4] s_max[0] = T.max(s_val[0], s_other_val[0]) s_val[0] = T.exp2(s_val[0] - s_max[0]) s_other_val[0] = T.exp2(s_other_val[0] - s_max[0]) scale[0] = s_val[0] / (s_val[0] + s_other_val[0]) other_scale[0] = s_other_val[0] / (s_val[0] + s_other_val[0]) for vec in T.vectorized(4): v_vec[vec] = V[bx, ty + by * 4, tx * 4 + vec] for vec in T.vectorized(4): v_other_vec[vec] = V_other[bx, ty + by * 4, tx * 4 + vec] for vec in range(4): v_vec[vec] = T.Cast("float16", T.Cast("float32", v_vec[vec]) * scale[0] + T.Cast("float32", v_other_vec[vec]) * other_scale[0]) for vec in T.vectorized(4): V[bx, ty + by * 4, tx * 4 + vec] = v_vec[vec] S[bx, ty + by * 4] = T.log2(s_val[0] + s_other_val[0]) + s_max[0] @T.prim_func def gemma_q0f16_multiply(var_input_embeds: T.handle, var_T_multiply: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() input_embeds = T.match_buffer(var_input_embeds, (batch_size, 1, 2048), "float16") T_multiply = T.match_buffer(var_T_multiply, (batch_size, 1, 2048), "float16") # with T.block("root"): for ax0_ax1_fused_0 in T.thread_binding(batch_size * 8, thread="blockIdx.x"): for ax0_ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_multiply"): v0 = T.axis.spatial(batch_size, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) // 2048) v1 = T.axis.spatial(2048, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) % 2048) T.reads(input_embeds[v0, 0, v1]) T.writes(T_multiply[v0, 0, v1]) T_multiply[v0, 0, v1] = input_embeds[v0, 0, v1] * T.float16(45.25) @T.prim_func def gemma_q0f16_multiply2(var_input_embeds: T.handle, var_T_multiply: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() input_embeds = T.match_buffer(var_input_embeds, (1, seq_len, 2048), "float16") T_multiply = T.match_buffer(var_T_multiply, (1, seq_len, 2048), "float16") # with T.block("root"): for ax0_ax1_fused_0 in T.thread_binding(seq_len * 8, thread="blockIdx.x"): for ax0_ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_multiply"): v0 = T.axis.spatial(seq_len, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) // 2048) v1 = T.axis.spatial(2048, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) % 2048) T.reads(input_embeds[0, v0, v1]) T.writes(T_multiply[0, v0, v1]) T_multiply[0, v0, v1] = input_embeds[0, v0, v1] * T.float16(45.25) @T.prim_func def gemma_q0f16_multiply4(input_embed: T.Buffer((1, 1, 2048), "float16"), T_multiply: T.Buffer((1, 1, 2048), "float16")): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) # with T.block("root"): for ax0_fused_0 in T.thread_binding(8, thread="blockIdx.x"): for ax0_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_multiply"): v0 = T.axis.spatial(2048, ax0_fused_0 * 256 + ax0_fused_1) T.reads(input_embed[0, 0, v0]) T.writes(T_multiply[0, 0, v0]) T_multiply[0, 0, v0] = input_embed[0, 0, v0] * T.float16(45.25) @T.prim_func def gemma_q0f16_reshape(var_lv: T.handle, var_T_reshape: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() lv = T.match_buffer(var_lv, (batch_size, 1, 2560), "float16") T_reshape = T.match_buffer(var_T_reshape, (batch_size, 1, 10, 256), "float16") # with T.block("root"): for ax0_ax1_ax2_fused_0 in T.thread_binding(batch_size * 10, thread="blockIdx.x"): for ax0_ax1_ax2_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_reshape"): v0 = T.axis.spatial(batch_size, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) // 2560) v1 = T.axis.spatial(10, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 2560 // 256) v2 = T.axis.spatial(256, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 256) T.reads(lv[v0, 0, v1 * 256 + v2]) T.writes(T_reshape[v0, 0, v1, v2]) T_reshape[v0, 0, v1, v2] = lv[v0, 0, v1 * 256 + v2] @T.prim_func def gemma_q0f16_reshape1(var_reshape216: T.handle, var_T_reshape: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() reshape216 = T.match_buffer(var_reshape216, (batch_size, 1, 10, 256), "float16") T_reshape = T.match_buffer(var_T_reshape, (batch_size, 10, 256), "float16") # with T.block("root"): for ax0_ax1_ax2_fused_0 in T.thread_binding(batch_size * 10, thread="blockIdx.x"): for ax0_ax1_ax2_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_reshape"): v0 = T.axis.spatial(batch_size, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) // 2560) v1 = T.axis.spatial(10, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 2560 // 256) v2 = T.axis.spatial(256, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 256) T.reads(reshape216[v0, 0, v1, v2]) T.writes(T_reshape[v0, v1, v2]) T_reshape[v0, v1, v2] = reshape216[v0, 0, v1, v2] @T.prim_func def gemma_q0f16_reshape2(var_lv55: T.handle, var_T_reshape: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() lv55 = T.match_buffer(var_lv55, (batch_size, 8, 256), "float16") T_reshape = T.match_buffer(var_T_reshape, (batch_size, 1, 8, 256), "float16") # with T.block("root"): for ax0_ax1_ax2_fused_0 in T.thread_binding(batch_size * 8, thread="blockIdx.x"): for ax0_ax1_ax2_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_reshape"): v0 = T.axis.spatial(batch_size, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) // 2048) v1 = T.axis.spatial(8, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 2048 // 256) v2 = T.axis.spatial(256, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 256) T.reads(lv55[v0, v1, v2]) T.writes(T_reshape[v0, 0, v1, v2]) T_reshape[v0, 0, v1, v2] = lv55[v0, v1, v2] @T.prim_func def gemma_q0f16_reshape3(var_reshape218: T.handle, var_T_reshape: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() reshape218 = T.match_buffer(var_reshape218, (batch_size, 1, 8, 256), "float16") T_reshape = T.match_buffer(var_T_reshape, (batch_size, 1, 2048), "float16") # with T.block("root"): for ax0_ax1_fused_0 in T.thread_binding(batch_size * 8, thread="blockIdx.x"): for ax0_ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_reshape"): v0 = T.axis.spatial(batch_size, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) // 2048) v1 = T.axis.spatial(2048, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) % 2048) T.reads(reshape218[v0, 0, v1 // 256, v1 % 256]) T.writes(T_reshape[v0, 0, v1]) T_reshape[v0, 0, v1] = reshape218[v0, 0, v1 // 256, v1 % 256] @T.prim_func def gemma_q0f16_reshape4(var_lv73: T.handle, var_T_reshape: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() lv73 = T.match_buffer(var_lv73, (1, seq_len, 2560), "float16") T_reshape = T.match_buffer(var_T_reshape, (1, seq_len, 10, 256), "float16") # with T.block("root"): for ax0_ax1_ax2_fused_0 in T.thread_binding(seq_len * 10, thread="blockIdx.x"): for ax0_ax1_ax2_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_reshape"): v0 = T.axis.spatial(seq_len, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) // 2560) v1 = T.axis.spatial(10, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 2560 // 256) v2 = T.axis.spatial(256, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 256) T.reads(lv73[0, v0, v1 * 256 + v2]) T.writes(T_reshape[0, v0, v1, v2]) T_reshape[0, v0, v1, v2] = lv73[0, v0, v1 * 256 + v2] @T.prim_func def gemma_q0f16_reshape5(var_reshape144: T.handle, var_T_reshape: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() reshape144 = T.match_buffer(var_reshape144, (1, seq_len, 10, 256), "float16") T_reshape = T.match_buffer(var_T_reshape, (seq_len, 10, 256), "float16") # with T.block("root"): for ax0_ax1_ax2_fused_0 in T.thread_binding(seq_len * 10, thread="blockIdx.x"): for ax0_ax1_ax2_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_reshape"): v0 = T.axis.spatial(seq_len, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) // 2560) v1 = T.axis.spatial(10, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 2560 // 256) v2 = T.axis.spatial(256, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 256) T.reads(reshape144[0, v0, v1, v2]) T.writes(T_reshape[v0, v1, v2]) T_reshape[v0, v1, v2] = reshape144[0, v0, v1, v2] @T.prim_func def gemma_q0f16_reshape6(var_lv37: T.handle, var_T_reshape: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() lv37 = T.match_buffer(var_lv37, (seq_len, 8, 256), "float16") T_reshape = T.match_buffer(var_T_reshape, (1, seq_len, 8, 256), "float16") # with T.block("root"): for ax0_ax1_ax2_fused_0 in T.thread_binding(seq_len * 8, thread="blockIdx.x"): for ax0_ax1_ax2_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_reshape"): v0 = T.axis.spatial(seq_len, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) // 2048) v1 = T.axis.spatial(8, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 2048 // 256) v2 = T.axis.spatial(256, (ax0_ax1_ax2_fused_0 * 256 + ax0_ax1_ax2_fused_1) % 256) T.reads(lv37[v0, v1, v2]) T.writes(T_reshape[0, v0, v1, v2]) T_reshape[0, v0, v1, v2] = lv37[v0, v1, v2] @T.prim_func def gemma_q0f16_reshape7(var_reshape146: T.handle, var_T_reshape: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() reshape146 = T.match_buffer(var_reshape146, (1, seq_len, 8, 256), "float16") T_reshape = T.match_buffer(var_T_reshape, (1, seq_len, 2048), "float16") # with T.block("root"): for ax0_ax1_fused_0 in T.thread_binding(seq_len * 8, thread="blockIdx.x"): for ax0_ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_reshape"): v0 = T.axis.spatial(seq_len, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) // 2048) v1 = T.axis.spatial(2048, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) % 2048) T.reads(reshape146[0, v0, v1 // 256, v1 % 256]) T.writes(T_reshape[0, v0, v1]) T_reshape[0, v0, v1] = reshape146[0, v0, v1 // 256, v1 % 256] @T.prim_func def gemma_q0f16_rms_norm(var_mul57: T.handle, model_layers_0_input_layernorm_weight4: T.Buffer((2048,), "float16"), var_T_cast: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size = T.int32() mul57 = T.match_buffer(var_mul57, (batch_size, 1, 2048), "float16") T_cast = T.match_buffer(var_T_cast, (batch_size, 1, 2048), "float16") # with T.block("root"): T_multiply_red_shared = T.alloc_buffer((batch_size, 1), scope="shared") T_multiply_red_rf_local = T.alloc_buffer((256, batch_size, 1), scope="local") for ax0_fused in T.thread_binding(batch_size, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 256, "pragma_unroll_explicit": 1}): with T.block("T_multiply_red_rf_init"): vax1_fused_1, v0 = T.axis.remap("SS", [ax1_fused_1, ax0_fused]) T.reads() T.writes(T_multiply_red_rf_local[vax1_fused_1, v0, 0]) T_multiply_red_rf_local[vax1_fused_1, v0, 0] = T.float32(0.0) for ax1_fused_0, u in T.grid(8, 1): with T.block("T_multiply_red_rf_update"): vax1_fused_1, v0, vax1_fused_0 = T.axis.remap("SSR", [ax1_fused_1, ax0_fused, ax1_fused_0]) T.reads(T_multiply_red_rf_local[vax1_fused_1, v0, 0], mul57[v0, 0, vax1_fused_0 * 256 + vax1_fused_1]) T.writes(T_multiply_red_rf_local[vax1_fused_1, v0, 0]) T_multiply_red_rf_local[vax1_fused_1, v0, 0] = T_multiply_red_rf_local[vax1_fused_1, v0, 0] + T.Cast("float32", mul57[v0, 0, vax1_fused_0 * 256 + vax1_fused_1]) * T.Cast("float32", mul57[v0, 0, vax1_fused_0 * 256 + vax1_fused_1]) for ax1_fused in range(1): for ax0 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_multiply_red"): vax1_fused_1, v0 = T.axis.remap("RS", [ax0, ax0_fused]) T.reads(T_multiply_red_rf_local[vax1_fused_1, v0, 0]) T.writes(T_multiply_red_shared[v0, 0]) with T.init(): T_multiply_red_shared[v0, 0] = T.float32(0.0) T_multiply_red_shared[v0, 0] = T_multiply_red_shared[v0, 0] + T_multiply_red_rf_local[vax1_fused_1, v0, 0] for ax0_ax1_fused_0 in range(8): for ax0_ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_cast_2"): v0 = T.axis.spatial(batch_size, ax0_fused) v1 = T.axis.spatial(2048, ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) T.reads(T_multiply_red_shared[v0, 0], mul57[v0, 0, v1], model_layers_0_input_layernorm_weight4[v1]) T.writes(T_cast[v0, 0, v1]) T_cast[v0, 0, v1] = T.Cast("float16", T.rsqrt(T_multiply_red_shared[v0, 0] * T.float32(0.00048828125) + T.float32(9.9999999999999995e-07)) * T.Cast("float32", mul57[v0, 0, v1]) * T.Cast("float32", model_layers_0_input_layernorm_weight4[v1])) @T.prim_func def gemma_q0f16_rms_norm1(var_mul38: T.handle, model_layers_0_input_layernorm_weight3: T.Buffer((2048,), "float16"), var_T_cast: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() mul38 = T.match_buffer(var_mul38, (1, seq_len, 2048), "float16") T_cast = T.match_buffer(var_T_cast, (1, seq_len, 2048), "float16") # with T.block("root"): T_multiply_red_shared = T.alloc_buffer((1, seq_len), scope="shared") T_multiply_red_rf_local = T.alloc_buffer((256, 1, seq_len), scope="local") for ax0_fused in T.thread_binding(seq_len, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 256, "pragma_unroll_explicit": 1}): with T.block("T_multiply_red_rf_init"): vax1_fused_1, v0 = T.axis.remap("SS", [ax1_fused_1, ax0_fused]) T.reads() T.writes(T_multiply_red_rf_local[vax1_fused_1, 0, v0]) T_multiply_red_rf_local[vax1_fused_1, 0, v0] = T.float32(0.0) for ax1_fused_0, u in T.grid(8, 1): with T.block("T_multiply_red_rf_update"): vax1_fused_1, v0, vax1_fused_0 = T.axis.remap("SSR", [ax1_fused_1, ax0_fused, ax1_fused_0]) T.reads(T_multiply_red_rf_local[vax1_fused_1, 0, v0], mul38[0, v0, vax1_fused_0 * 256 + vax1_fused_1]) T.writes(T_multiply_red_rf_local[vax1_fused_1, 0, v0]) T_multiply_red_rf_local[vax1_fused_1, 0, v0] = T_multiply_red_rf_local[vax1_fused_1, 0, v0] + T.Cast("float32", mul38[0, v0, vax1_fused_0 * 256 + vax1_fused_1]) * T.Cast("float32", mul38[0, v0, vax1_fused_0 * 256 + vax1_fused_1]) for ax1_fused in range(1): for ax0 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_multiply_red"): vax1_fused_1, v0 = T.axis.remap("RS", [ax0, ax0_fused]) T.reads(T_multiply_red_rf_local[vax1_fused_1, 0, v0]) T.writes(T_multiply_red_shared[0, v0]) with T.init(): T_multiply_red_shared[0, v0] = T.float32(0.0) T_multiply_red_shared[0, v0] = T_multiply_red_shared[0, v0] + T_multiply_red_rf_local[vax1_fused_1, 0, v0] for ax0_ax1_fused_0 in range(8): for ax0_ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_cast_2"): v0 = T.axis.spatial(seq_len, ax0_fused) v1 = T.axis.spatial(2048, ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) T.reads(T_multiply_red_shared[0, v0], mul38[0, v0, v1], model_layers_0_input_layernorm_weight3[v1]) T.writes(T_cast[0, v0, v1]) T_cast[0, v0, v1] = T.Cast("float16", T.rsqrt(T_multiply_red_shared[0, v0] * T.float32(0.00048828125) + T.float32(9.9999999999999995e-07)) * T.Cast("float32", mul38[0, v0, v1]) * T.Cast("float32", model_layers_0_input_layernorm_weight3[v1])) @T.prim_func def gemma_q0f16_rms_norm2(mul19: T.Buffer((1, 1, 2048), "float16"), model_layers_0_input_layernorm_weight2: T.Buffer((2048,), "float16"), T_cast: T.Buffer((1, 1, 2048), "float16")): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) # with T.block("root"): T_multiply_red_shared = T.alloc_buffer((1, 1), scope="shared") T_multiply_red_rf_local = T.alloc_buffer((256, 1, 1), scope="local") for ax0_fused in T.thread_binding(1, thread="blockIdx.x"): for ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x", annotations={"pragma_auto_unroll_max_step": 256, "pragma_unroll_explicit": 1}): with T.block("T_multiply_red_rf_init"): vax1_fused_1 = T.axis.spatial(256, ax1_fused_1) v0 = T.axis.spatial(1, 0) T.reads() T.writes(T_multiply_red_rf_local[vax1_fused_1, 0, 0]) T_multiply_red_rf_local[vax1_fused_1, 0, 0] = T.float32(0.0) for ax1_fused_0, u in T.grid(8, 1): with T.block("T_multiply_red_rf_update"): vax1_fused_1 = T.axis.spatial(256, ax1_fused_1) v0 = T.axis.spatial(1, 0) vax1_fused_0 = T.axis.reduce(8, ax1_fused_0) T.reads(T_multiply_red_rf_local[vax1_fused_1, 0, 0], mul19[0, 0, vax1_fused_0 * 256 + vax1_fused_1]) T.writes(T_multiply_red_rf_local[vax1_fused_1, 0, 0]) T_multiply_red_rf_local[vax1_fused_1, 0, 0] = T_multiply_red_rf_local[vax1_fused_1, 0, 0] + T.Cast("float32", mul19[0, 0, vax1_fused_0 * 256 + vax1_fused_1]) * T.Cast("float32", mul19[0, 0, vax1_fused_0 * 256 + vax1_fused_1]) for ax1_fused in range(1): for ax0 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_multiply_red"): vax1_fused_1 = T.axis.reduce(256, ax0) v0 = T.axis.spatial(1, 0) T.reads(T_multiply_red_rf_local[vax1_fused_1, 0, 0]) T.writes(T_multiply_red_shared[0, 0]) with T.init(): T_multiply_red_shared[0, 0] = T.float32(0.0) T_multiply_red_shared[0, 0] = T_multiply_red_shared[0, 0] + T_multiply_red_rf_local[vax1_fused_1, 0, 0] for ax0_fused_0 in range(8): for ax0_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_cast_2"): v0 = T.axis.spatial(2048, ax0_fused_0 * 256 + ax0_fused_1) T.reads(T_multiply_red_shared[0, 0], mul19[0, 0, v0], model_layers_0_input_layernorm_weight2[v0]) T.writes(T_cast[0, 0, v0]) T_cast[0, 0, v0] = T.Cast("float16", T.rsqrt(T_multiply_red_shared[0, 0] * T.float32(0.00048828125) + T.float32(9.9999999999999995e-07)) * T.Cast("float32", mul19[0, 0, v0]) * T.Cast("float32", model_layers_0_input_layernorm_weight2[v0])) @T.prim_func def gemma_q0f16_scatter_probs(var_src: T.handle, var_indices: T.handle, var_dst: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size, n = T.int32(is_size_var=True), T.int32(is_size_var=True) src = T.match_buffer(var_src, (batch_size, n)) indices = T.match_buffer(var_indices, (batch_size,), "int32") m = T.int32(is_size_var=True) dst = T.match_buffer(var_dst, (m, n)) # with T.block("root"): for ax0_ax1_fused_0 in T.thread_binding((batch_size * n + 255) // 256, thread="blockIdx.x"): for ax0_ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("scatter_2d"): v0 = T.axis.spatial(batch_size, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) % (n * batch_size) // n) v1 = T.axis.spatial(n, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) % n) T.where(ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1 < batch_size * n) T.reads(src[v0, v1], indices[v0]) T.writes(dst[indices[v0], v1]) dst[indices[v0], v1] = src[v0, v1] @T.prim_func def gemma_q0f16_shape_func(H: T.Buffer((T.int64(5),), "int64")): T.func_attr({"tir.is_host_func": 1}) H[T.int64(4)] = T.int64(512) * H[T.int64(1)] H[T.int64(2)] = T.int64(512) * ((H[T.int64(1)] + T.int64(4096) - T.int64(1)) // T.int64(4096)) H[T.int64(3)] = (H[T.int64(1)] + T.int64(4096) - T.int64(1)) // T.int64(4096) @T.prim_func def gemma_q0f16_shape_func1(H: T.Buffer((T.int64(4),), "int64")): T.func_attr({"tir.is_host_func": 1}) H[T.int64(3)] = T.int64(128) * H[T.int64(1)] * T.int64(4) @T.prim_func def gemma_q0f16_shape_func2(H: T.Buffer((T.int64(4),), "int64")): T.func_attr({"tir.is_host_func": 1}) H[T.int64(3)] = H[T.int64(1)] * T.int64(4) @T.prim_func def gemma_q0f16_shape_func3(H: T.Buffer((T.int64(5),), "int64")): T.func_attr({"tir.is_host_func": 1}) H[T.int64(4)] = T.int64(128) * H[T.int64(2)] * T.int64(4) @T.prim_func def gemma_q0f16_shape_func4(H: T.Buffer((T.int64(3),), "int64")): T.func_attr({"tir.is_host_func": 1}) H[T.int64(2)] = H[T.int64(0)] * T.int64(4) @T.prim_func def gemma_q0f16_shape_func5(H: T.Buffer((T.int64(4),), "int64")): T.func_attr({"tir.is_host_func": 1}) H[T.int64(3)] = T.int64(8192) * H[T.int64(1)] * T.int64(4) @T.prim_func def gemma_q0f16_softmax_with_chunked_sum(var_A: T.handle, var_temperature: T.handle, var_chunked_sum: T.handle, var_chunked_max: T.handle, var_softmax: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True) A = T.match_buffer(var_A, (batch_size, vocab_size)) temperature = T.match_buffer(var_temperature, (batch_size,)) num_chunks = T.int32(is_size_var=True) chunked_sum = T.match_buffer(var_chunked_sum, (batch_size, num_chunks)) chunked_max = T.match_buffer(var_chunked_max, (batch_size, num_chunks)) softmax = T.match_buffer(var_softmax, (batch_size, vocab_size)) # with T.block("root"): temp_max_shared = T.alloc_buffer((batch_size,), scope="shared") temp_sum_shared = T.alloc_buffer((batch_size,), scope="shared") for l0_l1_fused in T.thread_binding(batch_size * num_chunks, thread="blockIdx.x"): for ax0_1 in T.thread_binding(32, thread="threadIdx.x"): for ax0_0 in T.serial((num_chunks + 31) // 32, annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("max"): v0 = T.axis.spatial(batch_size, l0_l1_fused % (num_chunks * batch_size) // num_chunks) v1 = T.axis.reduce(num_chunks, ax0_0 * 32 + ax0_1) T.where(ax0_0 * 32 + ax0_1 < num_chunks) T.reads(chunked_max[v0, v1]) T.writes(temp_max_shared[v0]) with T.init(): temp_max_shared[v0] = T.float32(-340282346638528859811704183484516925440.0) temp_max_shared[v0] = T.max(temp_max_shared[v0], chunked_max[v0, v1]) for ax0_1 in T.thread_binding(32, thread="threadIdx.x"): for ax0_0 in T.serial((num_chunks + 31) // 32, annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): with T.block("sum_exp"): v0 = T.axis.spatial(batch_size, l0_l1_fused % (num_chunks * batch_size) // num_chunks) v1 = T.axis.reduce(num_chunks, ax0_0 * 32 + ax0_1) T.where(ax0_0 * 32 + ax0_1 < num_chunks) T.reads(temperature[v0], chunked_sum[v0, v1], chunked_max[v0, v1], temp_max_shared[v0]) T.writes(temp_sum_shared[v0]) with T.init(): temp_sum_shared[v0] = T.float32(0.0) temp_sum_shared[v0] = temp_sum_shared[v0] + T.Select(temperature[v0] > T.float32(1.0000000000000001e-05), T.exp(chunked_sum[v0, v1] + chunked_max[v0, v1] - temp_max_shared[v0]), T.Cast("float32", chunked_max[v0, v1] == temp_max_shared[v0]) * chunked_sum[v0, v1]) for l2_0 in T.serial(16, annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}): for l2_1 in T.thread_binding(8, thread="threadIdx.y"): for l2_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("log_pad"): v0 = T.axis.spatial(batch_size, l0_l1_fused % (num_chunks * batch_size) // num_chunks) v1 = T.axis.spatial(num_chunks, l0_l1_fused % num_chunks) v2 = T.axis.spatial(4096, l2_0 * 256 + l2_1 * 32 + l2_2) T.reads(temperature[v0], A[v0, v1 * 4096 + v2], temp_sum_shared[v0], temp_max_shared[v0]) T.writes(softmax[v0, v1 * 4096 + v2]) if v1 * 4096 + v2 < vocab_size: softmax[v0, v1 * 4096 + v2] = T.if_then_else(temperature[v0] > T.float32(1.0000000000000001e-05), T.exp(A[v0, v1 * 4096 + v2] / temperature[v0] - (T.log(temp_sum_shared[v0]) + temp_max_shared[v0])), T.Cast("float32", A[v0, v1 * 4096 + v2] == temp_max_shared[v0]) / temp_sum_shared[v0]) @T.prim_func def gemma_q0f16_take(var_rms_norm110: T.handle, var_logit_positions: T.handle, var_T_take: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) seq_len = T.int32() rms_norm110 = T.match_buffer(var_rms_norm110, (1, seq_len, 2048), "float16") batch_size = T.int32() logit_positions = T.match_buffer(var_logit_positions, (batch_size,), "int32") T_take = T.match_buffer(var_T_take, (1, batch_size, 2048), "float16") # with T.block("root"): for ax0_ax1_fused_0 in T.thread_binding(batch_size * 8, thread="blockIdx.x"): for ax0_ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_take"): v0 = T.axis.spatial(batch_size, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) // 2048) v1 = T.axis.spatial(2048, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) % 2048) T.reads(rms_norm110[0, logit_positions[v0], v1], logit_positions[v0]) T.writes(T_take[0, v0, v1]) T_take[0, v0, v1] = rms_norm110[0, logit_positions[v0], v1] @T.prim_func def gemma_q0f16_take1(var_model_embed_tokens_weight: T.handle, var_input_ids: T.handle, var_T_take: T.handle): T.func_attr({"tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) vocab_size = T.int32() model_embed_tokens_weight = T.match_buffer(var_model_embed_tokens_weight, (vocab_size, 2048), "float16") seq_len = T.int32() input_ids = T.match_buffer(var_input_ids, (seq_len,), "int32") T_take = T.match_buffer(var_T_take, (seq_len, 2048), "float16") # with T.block("root"): for ax0_ax1_fused_0 in T.thread_binding(seq_len * 8, thread="blockIdx.x"): for ax0_ax1_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("T_take"): v0 = T.axis.spatial(seq_len, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) // 2048) v1 = T.axis.spatial(2048, (ax0_ax1_fused_0 * 256 + ax0_ax1_fused_1) % 2048) T.reads(model_embed_tokens_weight[input_ids[v0], v1], input_ids[v0]) T.writes(T_take[v0, v1]) T_take[v0, v1] = model_embed_tokens_weight[input_ids[v0], v1] @T.prim_func def gemma_q0f16_tir_kv_cache_debug_get_kv(var_pages: T.handle, var_position_map: T.handle, var_k_data: T.handle, var_v_data: T.handle, layer_id: T.int32): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) num_pages, page_size = T.int32(), T.int32(is_size_var=True) pages = T.match_buffer(var_pages, (num_pages, 2, 1, page_size, 256), "float16", offset_factor=1) seqlen = T.int32(is_size_var=True) position_map = T.match_buffer(var_position_map, (seqlen,), "int32", offset_factor=1) k_data = T.match_buffer(var_k_data, (18, seqlen, 1, 256), "float16") v_data = T.match_buffer(var_v_data, (18, seqlen, 1, 256), "float16") # with T.block("root"): for p_h_d_fused_0 in T.thread_binding(seqlen, thread="blockIdx.x"): for p_h_d_fused_1 in T.thread_binding(256, thread="threadIdx.x"): with T.block("copy0"): vp = T.axis.spatial(seqlen, (p_h_d_fused_0 * 256 + p_h_d_fused_1) // 256) vh = T.axis.spatial(1, 0) vd = T.axis.spatial(256, (p_h_d_fused_0 * 256 + p_h_d_fused_1) % 256) T.reads(position_map[vp], pages[position_map[vp] // page_size, 0:2, vh, position_map[vp] % page_size, vd]) T.writes(k_data[layer_id, vp, vh, vd], v_data[layer_id, vp, vh, vd]) position: T.int32 = position_map[vp] k_data[layer_id, vp, vh, vd] = pages[position // page_size, 0, vh, position % page_size, vd] v_data[layer_id, vp, vh, vd] = pages[position // page_size, 1, vh, position % page_size, vd] @T.prim_func def gemma_q0f16_tir_kv_cache_transpose_append(var_pages: T.handle, var_k_data: T.handle, var_v_data: T.handle, var_position_map: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1, "tir.noalias": T.bool(True)}) num_pages = T.int32() pages = T.match_buffer(var_pages, (num_pages, 2, 1, 16, 256), "float16", offset_factor=1) ntoken = T.int32(is_size_var=True) k_data = T.match_buffer(var_k_data, (ntoken, 1, 256), "float16") v_data = T.match_buffer(var_v_data, (ntoken, 1, 256), "float16") position_map = T.match_buffer(var_position_map, (ntoken,), "int32", offset_factor=1) # with T.block("root"): for global_pos_h_f_fused_0 in T.thread_binding(ntoken, thread="blockIdx.x"): for global_pos_h_f_fused_1 in T.thread_binding(256, thread="threadIdx.x"): if position_map[(global_pos_h_f_fused_0 * 256 + global_pos_h_f_fused_1) // 256] != -1: with T.block("k_transpose_append"): vgpos = T.axis.spatial(ntoken, (global_pos_h_f_fused_0 * 256 + global_pos_h_f_fused_1) // 256) vh = T.axis.spatial(1, 0) vf = T.axis.spatial(256, (global_pos_h_f_fused_0 * 256 + global_pos_h_f_fused_1) % 256) T.reads(position_map[vgpos], k_data[vgpos, vh, vf]) T.writes(pages[position_map[vgpos] // 16, 0, vh, position_map[vgpos] % 16, vf]) position: T.int32 = position_map[vgpos] pages[position // 16, 0, vh, position % 16, vf] = k_data[vgpos, vh, vf] with T.block("v_transpose_append"): vgpos = T.axis.spatial(ntoken, (global_pos_h_f_fused_0 * 256 + global_pos_h_f_fused_1) // 256) vh = T.axis.spatial(1, 0) vf = T.axis.spatial(256, (global_pos_h_f_fused_0 * 256 + global_pos_h_f_fused_1) % 256) T.reads(position_map[vgpos], v_data[vgpos, vh, vf]) T.writes(pages[position_map[vgpos] // 16, 1, vh, position_map[vgpos] % 16, vf]) position: T.int32 = position_map[vgpos] pages[position // 16, 1, vh, position % 16, vf] = v_data[vgpos, vh, vf] @T.prim_func def gemma_q0f16_tree_attn_paged_kv(_0: T.int32, var_q: T.handle, var_q_indptr: T.handle, var_pages: T.handle, var_page_indptr: T.handle, var_page_values: T.handle, var_length_info: T.handle, var_k_rope_pos_offset: T.handle, var_q_rope_position: T.handle, var_output: T.handle, var_lse: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32, tree_order_indptr_handle: T.handle, tree_order_handle: T.handle): T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1}) total_len = T.int32(is_size_var=True) q = T.match_buffer(var_q, (total_len, 8, 256), "float16") batch_size = T.int32(is_size_var=True) q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1) max_num_pages = T.int32(is_size_var=True) pages = T.match_buffer(var_pages, (max_num_pages, 2, 1, 32, 256), "float16") page_indptr = T.match_buffer(var_page_indptr, (batch_size + 1,), "int32", offset_factor=1) nnz_pages = T.int32(is_size_var=True) page_values = T.match_buffer(var_page_values, (nnz_pages,), "int32", offset_factor=1) length_info = T.match_buffer(var_length_info, (batch_size,), "int32", offset_factor=1) k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1) q_rope_position = T.match_buffer(var_q_rope_position, (total_len,), "int32", offset_factor=1) output = T.match_buffer(var_output, (total_len, 8, 256), "float16") lse = T.match_buffer(var_lse, (total_len, 8)) tree_order_indptr = T.match_buffer(tree_order_indptr_handle, (batch_size + 1,), "int32", offset_factor=1) total_tree_order_len = T.int32(is_size_var=True) tree_order = T.match_buffer(tree_order_handle, (total_tree_order_len, 2), "int32", offset_factor=1) # with T.block("root"): assert rotary_mode == 0, "Inline rotary mode is not supported in tree attention." for lbx in T.thread_binding(16, thread="blockIdx.x"): for lby in T.thread_binding(1, thread="blockIdx.y"): for lty in T.thread_binding(4, thread="threadIdx.y"): for ltx in T.thread_binding(32, thread="threadIdx.x"): with T.block("attn"): bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx]) T.reads() T.writes() tile_id = T.alloc_buffer((1,), "int32", scope="local") batch_idx = T.alloc_buffer((1,), "int32", scope="local") batch_tiles = T.alloc_buffer((1,), "int32", scope="local") batch_rows = T.alloc_buffer((1,), "int32", scope="local") iterator = T.alloc_buffer((1,), "int32", scope="local") kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local") Q_smem = T.alloc_buffer((16, 256), "float16", scope="shared") K_smem = T.alloc_buffer((32, 256), "float16", scope="shared") V_smem = T.alloc_buffer((32, 256), "float16", scope="shared") S_smem = T.alloc_buffer((16, 32), scope="shared") S_local = T.alloc_buffer((16, 32), scope="local") O_local = T.alloc_buffer((16, 256), scope="local") m_smem = T.alloc_buffer((16,), scope="shared") m_prev_smem = T.alloc_buffer((16,), scope="shared") d_smem = T.alloc_buffer((16,), scope="shared") m_new = T.alloc_buffer((1,), scope="local") m_prev = T.alloc_buffer((1,), scope="local") d_new = T.alloc_buffer((1,), scope="local") tile_id[0] = bx batch_idx[0] = 0 batch_rows[0] = (q_indptr[1] - q_indptr[0]) * 8 batch_tiles[0] = (batch_rows[0] + 16 - 1) // 16 while T.tvm_thread_invariant(batch_idx[0] < batch_size): while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size: tile_id[0] = tile_id[0] - batch_tiles[0] batch_idx[0] = batch_idx[0] + 1 if batch_idx[0] < batch_size: b_idx: T.int32 = batch_idx[0] batch_rows[0] = (q_indptr[b_idx + 1] - q_indptr[b_idx]) * 8 batch_tiles[0] = (batch_rows[0] + 16 - 1) // 16 if T.tvm_thread_invariant(batch_idx[0] < batch_size): b_idx: T.int32 = batch_idx[0] LH_start: T.int32 = tile_id[0] * 16 q_indptr_val: T.int32 = q_indptr[b_idx] cur_page_indptr_begin: T.int32 = page_indptr[b_idx] cur_page_indptr_end: T.int32 = page_indptr[b_idx + 1] kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 32 + length_info[b_idx], 0) T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: m_smem[row] = T.float32(-50000.0) d_smem[row] = T.float32(1.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 8): with T.block("O_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) T.reads() T.writes(O_local[i, j]) O_local[i, j] = T.float32(0.0) T.tvm_storage_sync("shared") for li_lj_fused_0 in range(8): for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for li_lj_fused_3 in T.vectorized(4): with T.block("Q_load"): i = T.axis.spatial(16, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 256) j = T.axis.spatial(256, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = q_indptr_val + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: freq = T.float32() Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 128, q[cur_L, cur_H_qo, j + 128] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 128]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 256) / T.float32(256.0))}), q[cur_L, cur_H_qo, j]) else: Q_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for iterator_1 in range((kv_chunk_len[0] + 31) // 32): L_kv_start: T.int32 = iterator_1 * 32 for lz_ly_fused_0 in range(16): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("K_load"): i = T.axis.spatial(32, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 256) j = T.axis.spatial(256, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = cur_L page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 32] page_offset: T.int32 = seq_offset % 32 K_smem[i, j] = pages[page_no, 0, by, page_offset, j] else: K_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") for lz_ly_fused_0 in range(16): for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"): for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"): for lz_ly_fused_3 in T.vectorized(4): with T.block("V_load"): i = T.axis.spatial(32, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 256) j = T.axis.spatial(256, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 256) T.reads() T.writes() cur_L: T.int32 = L_kv_start + i if cur_L < kv_chunk_len[0]: seq_offset: T.int32 = cur_L page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 32] page_offset: T.int32 = seq_offset % 32 V_smem[i, j] = pages[page_no, 1, by, page_offset, j] else: V_smem[i, j] = T.float16(0.0) T.tvm_storage_sync("shared") with T.block(""): T.reads(Q_smem[0:16, 0:256], K_smem[0:32, 0:256]) T.writes(S_local[0:16, 0:32]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(2, 2): with T.block("S_gemm_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 2 + li_1_init) j = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 2 + lj_1_init) T.reads() T.writes(S_local[i, j]) S_local[i, j] = T.float32(0.0) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, li_1, lj_1, lk_1 in T.grid(32, 2, 2, 8): with T.block("S_gemm_update"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 2 + li_1) j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 2 + lj_1) k = T.axis.reduce(256, lk_0 * 8 + lk_1) T.reads(S_local[i, j], Q_smem[i, k], K_smem[j, k]) T.writes(S_local[i, j]) S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k]) * T.Cast("float32", K_smem[j, k]) * attn_score_scaling_factor * T.float32(0.090168440055560212) T.tvm_storage_sync("shared") for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(2, 2): with T.block("S_store"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 2 + li_1) j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 2 + lj_1) T.reads(S_local[i, j]) T.writes(S_smem[i, j]) S_smem[i, j] = S_local[i, j] T.tvm_storage_sync("shared") for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: with T.block("update1"): T.reads(m_smem[row], kv_chunk_len[0], tree_order_indptr[b_idx:b_idx + 2], tree_order[T.min((LH_start + row) // 8 + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]):T.min((LH_start + row) // 8 + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max((LH_start + row) // 8 + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min((LH_start + row) // 8 + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:32], d_smem[row], m_prev[i]) T.writes(m_prev[i], m_new[i], d_new[i]) m_prev[i] = m_smem[row] m_new[i] = m_smem[row] row_: T.int32 = (LH_start + row) // 8 for j in range(32): if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) or tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 0] and tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 1]): m_new[i] = T.max(m_new[i], S_smem[row, j]) d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx with T.block("update"): T.reads(kv_chunk_len[0], tree_order_indptr[b_idx:b_idx + 2], tree_order[T.min((LH_start + row) // 8 + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]):T.min((LH_start + row) // 8 + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max((LH_start + row) // 8 + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min((LH_start + row) // 8 + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:32], m_new[i]) T.writes(S_smem[row, 0:32]) for j in range(32): if row < 16: row_: T.int32 = (LH_start + row) // 8 if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) or tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 0] and tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 1]): S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i]) else: S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i]) for i in range(1): row: T.int32 = i * 32 * 4 + ty * 32 + tx if row < 16: with T.block("update"): T.reads(d_new[i], S_smem[row, 0:32], m_new[i], m_prev[i]) T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row]) for j in range(32): d_new[i] = d_new[i] + S_smem[row, j] m_smem[row] = m_new[i] d_smem[row] = d_new[i] m_prev_smem[row] = m_prev[i] T.tvm_storage_sync("shared") with T.block(""): T.reads(m_prev_smem[0:16], m_smem[0:16], S_smem[0:16, 0:32], V_smem[0:32, 0:256]) T.writes(O_local[0:16, 0:256]) for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"): for li_1_init, lj_1_init in T.grid(4, 8): with T.block("O_gemm_init"): i = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 32 * 4 + li_1_init) j = T.axis.spatial(256, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 32 * 8 + lj_1_init) T.reads() T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for lk_0, lk_1, li_1, lj_1 in T.grid(4, 8, 4, 8): with T.block("O_gemm_update"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) k = T.axis.reduce(32, lk_0 * 8 + lk_1) T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k], V_smem[k, j]) T.writes(O_local[i, j]) O_local[i, j] = O_local[i, j] + S_smem[i, k] * T.Cast("float32", V_smem[k, j]) for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"): for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"): for li_1, lj_1 in T.grid(4, 8): with T.block("O_store"): i = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 32 * 4 + li_1) j = T.axis.spatial(256, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 32 * 8 + lj_1) T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i]) T.writes(output[q_indptr[b_idx] + (LH_start + i) // 8, by * 8 + (LH_start + i) % 8, j]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i]) for li_0 in range(1): for li_1 in T.thread_binding(4, thread="threadIdx.y"): for li_2 in T.thread_binding(32, thread="threadIdx.x"): with T.block("lse_store"): i = T.axis.spatial(16, li_0 * 128 + li_1 * 32 + li_2) T.where((li_0 * 4 + li_1) * 32 + li_2 < 16) T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i]) T.writes(lse[q_indptr[b_idx] + (LH_start + i) // 8, by * 8 + (LH_start + i) % 8]) cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i) // 8 cur_H_qo: T.int32 = by * 8 + (LH_start + i) % 8 if cur_L < q_indptr[b_idx + 1]: lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i]) tile_id[0] = tile_id[0] + 16 @R.function def _metadata() -> R.Object: shape_heap: R.Object = R.null_value() return R.str("{\"model_type\": \"gemma\", \"quantization\": \"q0f16\", \"context_window_size\": 8192, \"sliding_window_size\": -1, \"attention_sink_size\": -1, \"prefill_chunk_size\": 8192, \"tensor_parallel_shards\": 1, \"pipeline_parallel_stages\": 1, \"disaggregation\": false, \"kv_state_kind\": \"kv_cache\", \"max_batch_size\": 128, \"params\": [{\"name\": \"model.embed_tokens.weight\", \"shape\": [\"vocab_size\", 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.0.self_attn.qkv_proj.weight\", \"shape\": [2560, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.0.self_attn.o_proj.weight\", \"shape\": [2048, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.0.mlp.gate_up_proj.weight\", \"shape\": [32768, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.0.mlp.down_proj.weight\", \"shape\": [2048, 16384], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.0.input_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.0.post_attention_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.1.self_attn.qkv_proj.weight\", \"shape\": [2560, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.1.self_attn.o_proj.weight\", \"shape\": [2048, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.1.mlp.gate_up_proj.weight\", \"shape\": [32768, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.1.mlp.down_proj.weight\", \"shape\": [2048, 16384], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.1.input_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.1.post_attention_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.2.self_attn.qkv_proj.weight\", \"shape\": [2560, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.2.self_attn.o_proj.weight\", \"shape\": [2048, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.2.mlp.gate_up_proj.weight\", \"shape\": [32768, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.2.mlp.down_proj.weight\", \"shape\": [2048, 16384], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.2.input_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.2.post_attention_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.3.self_attn.qkv_proj.weight\", \"shape\": [2560, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.3.self_attn.o_proj.weight\", \"shape\": [2048, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.3.mlp.gate_up_proj.weight\", \"shape\": [32768, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.3.mlp.down_proj.weight\", \"shape\": [2048, 16384], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.3.input_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.3.post_attention_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.4.self_attn.qkv_proj.weight\", \"shape\": [2560, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.4.self_attn.o_proj.weight\", \"shape\": [2048, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.4.mlp.gate_up_proj.weight\", \"shape\": [32768, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.4.mlp.down_proj.weight\", \"shape\": [2048, 16384], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.4.input_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.4.post_attention_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.5.self_attn.qkv_proj.weight\", \"shape\": [2560, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.5.self_attn.o_proj.weight\", \"shape\": [2048, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.5.mlp.gate_up_proj.weight\", \"shape\": [32768, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.5.mlp.down_proj.weight\", \"shape\": [2048, 16384], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.5.input_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.5.post_attention_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.6.self_attn.qkv_proj.weight\", \"shape\": [2560, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.6.self_attn.o_proj.weight\", \"shape\": [2048, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.6.mlp.gate_up_proj.weight\", \"shape\": [32768, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.6.mlp.down_proj.weight\", \"shape\": [2048, 16384], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.6.input_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.6.post_attention_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.7.self_attn.qkv_proj.weight\", \"shape\": [2560, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.7.self_attn.o_proj.weight\", \"shape\": [2048, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.7.mlp.gate_up_proj.weight\", \"shape\": [32768, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.7.mlp.down_proj.weight\", \"shape\": [2048, 16384], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.7.input_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.7.post_attention_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.8.self_attn.qkv_proj.weight\", \"shape\": [2560, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.8.self_attn.o_proj.weight\", \"shape\": [2048, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.8.mlp.gate_up_proj.weight\", \"shape\": [32768, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.8.mlp.down_proj.weight\", \"shape\": [2048, 16384], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.8.input_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.8.post_attention_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.9.self_attn.qkv_proj.weight\", \"shape\": [2560, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.9.self_attn.o_proj.weight\", \"shape\": [2048, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.9.mlp.gate_up_proj.weight\", \"shape\": [32768, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.9.mlp.down_proj.weight\", \"shape\": [2048, 16384], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.9.input_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.9.post_attention_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.10.self_attn.qkv_proj.weight\", \"shape\": [2560, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.10.self_attn.o_proj.weight\", \"shape\": [2048, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.10.mlp.gate_up_proj.weight\", \"shape\": [32768, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.10.mlp.down_proj.weight\", \"shape\": [2048, 16384], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.10.input_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.10.post_attention_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.11.self_attn.qkv_proj.weight\", \"shape\": [2560, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.11.self_attn.o_proj.weight\", \"shape\": [2048, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.11.mlp.gate_up_proj.weight\", \"shape\": [32768, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.11.mlp.down_proj.weight\", \"shape\": [2048, 16384], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.11.input_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.11.post_attention_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.12.self_attn.qkv_proj.weight\", \"shape\": [2560, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.12.self_attn.o_proj.weight\", \"shape\": [2048, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.12.mlp.gate_up_proj.weight\", \"shape\": [32768, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.12.mlp.down_proj.weight\", \"shape\": [2048, 16384], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.12.input_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.12.post_attention_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.13.self_attn.qkv_proj.weight\", \"shape\": [2560, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.13.self_attn.o_proj.weight\", \"shape\": [2048, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.13.mlp.gate_up_proj.weight\", \"shape\": [32768, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.13.mlp.down_proj.weight\", \"shape\": [2048, 16384], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.13.input_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.13.post_attention_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.14.self_attn.qkv_proj.weight\", \"shape\": [2560, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.14.self_attn.o_proj.weight\", \"shape\": [2048, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.14.mlp.gate_up_proj.weight\", \"shape\": [32768, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.14.mlp.down_proj.weight\", \"shape\": [2048, 16384], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.14.input_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.14.post_attention_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.15.self_attn.qkv_proj.weight\", \"shape\": [2560, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.15.self_attn.o_proj.weight\", \"shape\": [2048, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.15.mlp.gate_up_proj.weight\", \"shape\": [32768, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.15.mlp.down_proj.weight\", \"shape\": [2048, 16384], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.15.input_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.15.post_attention_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.16.self_attn.qkv_proj.weight\", \"shape\": [2560, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.16.self_attn.o_proj.weight\", \"shape\": [2048, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.16.mlp.gate_up_proj.weight\", \"shape\": [32768, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.16.mlp.down_proj.weight\", \"shape\": [2048, 16384], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.16.input_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.16.post_attention_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.17.self_attn.qkv_proj.weight\", \"shape\": [2560, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.17.self_attn.o_proj.weight\", \"shape\": [2048, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.17.mlp.gate_up_proj.weight\", \"shape\": [32768, 2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.17.mlp.down_proj.weight\", \"shape\": [2048, 16384], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.17.input_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.layers.17.post_attention_layernorm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}, {\"name\": \"model.norm.weight\", \"shape\": [2048], \"dtype\": \"float16\", \"preprocs\": [], \"pipeline_stages\": [0]}], \"kv_cache\": {\"num_hidden_layers\": 18, \"num_attention_heads\": 8, \"num_key_value_heads\": 1, \"head_dim\": 256}, \"memory_usage\": {\"alloc_embedding_tensor\": 33554432, \"batch_decode\": 17301504, \"batch_prefill\": 1107820544, \"batch_verify\": 1107296256, \"create_tir_paged_kv_cache\": 0, \"decode\": 135168, \"embed\": 33554432, \"prefill\": 1107300352, \"softmax_with_temperature\": 0}}") @R.function def alloc_embedding_tensor() -> R.Tensor((8192, 2048), dtype="float16"): R.func_attr({"relax.force_pure": True}) shape_heap: R.Object = R.null_value() storage: R.Object = R.vm.alloc_storage(R.shape([33554432]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv: R.Tensor((8192, 2048), dtype="float16") = R.vm.alloc_tensor(storage, R.prim_value(0), R.shape([8192, 2048]), R.dtype("float16")) R.vm.kill_object(storage) return gv @R.function def batch_decode(input_embeds: R.Tensor(("batch_size", 1, 2048), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 2048), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"))) -> R.Tuple(R.Tensor(("batch_size", 1, "vocab_size"), dtype="float32"), R.Object): batch_size = T.int64() vocab_size = T.int64() R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.force_pure": True, "relax.rewrite_cuda_graph.capture_symbolic_vars": ["batch_size"], "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 8192, "total_seq_len": 8192}}) cls = Module shape_heap: R.Tensor(dtype="int64", ndim=1) = R.call_builtin_with_ctx("vm.builtin.alloc_shape_heap", (R.prim_value(4),), sinfo_args=(R.Tensor(dtype="int64", ndim=1),)) R.call_packed("vm.builtin.check_tensor_info", input_embeds, R.prim_value(3), R.dtype("float16"), R.str("ErrorContext(fn=batch_decode, loc=param[0], param=input_embeds, annotation=R.Tensor((batch_size, 1, 2048), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_tuple_info", packed_params, R.prim_value(110), R.str("ErrorContext(fn=batch_decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 2048), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv8: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[0] R.call_packed("vm.builtin.check_tensor_info", gv8, R.prim_value(2), R.dtype("float16"), R.str("ErrorContext(fn=batch_decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 2048), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", input_embeds, shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), R.str("ErrorContext(fn=batch_decode, loc=param[0], param=input_embeds, annotation=R.Tensor((batch_size, 1, 2048), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv8, shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(2048), R.str("ErrorContext(fn=batch_decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 2048), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) cls.gemma_q0f16_shape_func1(shape_heap) storage3: R.Object = R.vm.alloc_storage(R.shape([524288]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv9: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc3: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv9, R.dtype("float16")) cls.gemma_q0f16_multiply(input_embeds, alloc3) model_layers_0_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[5] storage4: R.Object = R.vm.alloc_storage(R.shape([4194304]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv10: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc4: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv10, R.dtype("float16")) cls.gemma_q0f16_rms_norm(alloc3, model_layers_0_input_layernorm_weight4, alloc4) R.vm.kill_object(model_layers_0_input_layernorm_weight4) model_layers_0_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[1] storage5: R.Object = R.vm.alloc_storage(R.shape([8388608]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv11: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc5: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv11, R.dtype("float16")) cls.gemma_q0f16_NT_matmul(alloc4, model_layers_0_self_attn_qkv_proj_weight4, alloc5) R.vm.kill_object(alloc4) R.vm.kill_object(model_layers_0_self_attn_qkv_proj_weight4) gv12: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape216: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc5, gv12, sinfo_args=(R.Tensor((batch_size, 1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc5) gv13: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape217: R.Tensor((batch_size, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape216, gv13, sinfo_args=(R.Tensor((batch_size, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape216) gv14: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc6: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv14, R.dtype("float16")) _5: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape217, alloc6) R.vm.kill_object(reshape217) gv15: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape218: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc6, gv15, sinfo_args=(R.Tensor((batch_size, 1, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc6) gv16: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape219: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape218, gv16, sinfo_args=(R.Tensor((batch_size, 1, 2048), dtype="float16"),)) R.vm.kill_object(reshape218) model_layers_0_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[2] gv17: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc7: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv17, R.dtype("float16")) cls.gemma_q0f16_NT_matmul1(reshape219, model_layers_0_self_attn_o_proj_weight4, alloc7) R.vm.kill_object(reshape219) R.vm.kill_object(model_layers_0_self_attn_o_proj_weight4) model_layers_0_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[6] gv18: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc8: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv18, R.dtype("float16")) storage6: R.Object = R.vm.alloc_storage(R.shape([4194304]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv19: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc9: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv19, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc7, alloc3, model_layers_0_post_attention_layernorm_weight4, alloc8, alloc9) R.vm.kill_object(alloc3) R.vm.kill_object(alloc7) R.vm.kill_object(model_layers_0_post_attention_layernorm_weight4) lv: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc8, alloc9 model_layers_0_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[3] gv20: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc10: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv20, R.dtype("float16")) cls.gemma_q0f16_NT_matmul2(alloc8, model_layers_0_mlp_gate_up_proj_weight4, alloc10) R.vm.kill_object(alloc8) R.vm.kill_object(model_layers_0_mlp_gate_up_proj_weight4) gv21: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc11: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv21, R.dtype("float16")) cls.gemma_q0f16_fused_split_gelu_tanh_multiply1(alloc10, alloc11) R.vm.kill_object(alloc10) model_layers_0_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[4] gv22: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc12: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv22, R.dtype("float16")) cls.gemma_q0f16_NT_matmul3(alloc11, model_layers_0_mlp_down_proj_weight4, alloc12) R.vm.kill_object(alloc11) R.vm.kill_object(model_layers_0_mlp_down_proj_weight4) model_layers_1_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[11] gv23: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc13: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv23, R.dtype("float16")) gv24: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc14: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv24, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc12, alloc9, model_layers_1_input_layernorm_weight4, alloc13, alloc14) R.vm.kill_object(alloc9) R.vm.kill_object(alloc12) R.vm.kill_object(model_layers_1_input_layernorm_weight4) lv2: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc13, alloc14 model_layers_1_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[7] gv25: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc15: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv25, R.dtype("float16")) cls.gemma_q0f16_NT_matmul(alloc13, model_layers_1_self_attn_qkv_proj_weight4, alloc15) R.vm.kill_object(alloc13) R.vm.kill_object(model_layers_1_self_attn_qkv_proj_weight4) gv26: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape220: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc15, gv26, sinfo_args=(R.Tensor((batch_size, 1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc15) gv27: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape221: R.Tensor((batch_size, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape220, gv27, sinfo_args=(R.Tensor((batch_size, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape220) gv28: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc16: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv28, R.dtype("float16")) _13: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape221, alloc16) R.vm.kill_object(reshape221) gv29: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape222: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc16, gv29, sinfo_args=(R.Tensor((batch_size, 1, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc16) gv30: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape223: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape222, gv30, sinfo_args=(R.Tensor((batch_size, 1, 2048), dtype="float16"),)) R.vm.kill_object(reshape222) model_layers_1_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[8] gv31: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc17: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv31, R.dtype("float16")) cls.gemma_q0f16_NT_matmul1(reshape223, model_layers_1_self_attn_o_proj_weight4, alloc17) R.vm.kill_object(reshape223) R.vm.kill_object(model_layers_1_self_attn_o_proj_weight4) model_layers_1_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[12] gv32: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc18: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv32, R.dtype("float16")) gv33: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc19: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv33, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc17, alloc14, model_layers_1_post_attention_layernorm_weight4, alloc18, alloc19) R.vm.kill_object(alloc14) R.vm.kill_object(alloc17) R.vm.kill_object(model_layers_1_post_attention_layernorm_weight4) lv4: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc18, alloc19 model_layers_1_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[9] gv34: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc20: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv34, R.dtype("float16")) cls.gemma_q0f16_NT_matmul2(alloc18, model_layers_1_mlp_gate_up_proj_weight4, alloc20) R.vm.kill_object(alloc18) R.vm.kill_object(model_layers_1_mlp_gate_up_proj_weight4) gv35: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc21: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv35, R.dtype("float16")) cls.gemma_q0f16_fused_split_gelu_tanh_multiply1(alloc20, alloc21) R.vm.kill_object(alloc20) model_layers_1_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[10] gv36: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc22: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv36, R.dtype("float16")) cls.gemma_q0f16_NT_matmul3(alloc21, model_layers_1_mlp_down_proj_weight4, alloc22) R.vm.kill_object(alloc21) R.vm.kill_object(model_layers_1_mlp_down_proj_weight4) model_layers_2_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[17] gv37: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc23: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv37, R.dtype("float16")) gv38: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc24: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv38, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc22, alloc19, model_layers_2_input_layernorm_weight4, alloc23, alloc24) R.vm.kill_object(alloc19) R.vm.kill_object(alloc22) R.vm.kill_object(model_layers_2_input_layernorm_weight4) lv6: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc23, alloc24 model_layers_2_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[13] gv39: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc25: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv39, R.dtype("float16")) cls.gemma_q0f16_NT_matmul(alloc23, model_layers_2_self_attn_qkv_proj_weight4, alloc25) R.vm.kill_object(alloc23) R.vm.kill_object(model_layers_2_self_attn_qkv_proj_weight4) gv40: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape224: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc25, gv40, sinfo_args=(R.Tensor((batch_size, 1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc25) gv41: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape225: R.Tensor((batch_size, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape224, gv41, sinfo_args=(R.Tensor((batch_size, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape224) gv42: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc26: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv42, R.dtype("float16")) _21: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape225, alloc26) R.vm.kill_object(reshape225) gv43: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape226: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc26, gv43, sinfo_args=(R.Tensor((batch_size, 1, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc26) gv44: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape227: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape226, gv44, sinfo_args=(R.Tensor((batch_size, 1, 2048), dtype="float16"),)) R.vm.kill_object(reshape226) model_layers_2_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[14] gv45: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc27: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv45, R.dtype("float16")) cls.gemma_q0f16_NT_matmul1(reshape227, model_layers_2_self_attn_o_proj_weight4, alloc27) R.vm.kill_object(reshape227) R.vm.kill_object(model_layers_2_self_attn_o_proj_weight4) model_layers_2_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[18] gv46: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc28: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv46, R.dtype("float16")) gv47: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc29: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv47, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc27, alloc24, model_layers_2_post_attention_layernorm_weight4, alloc28, alloc29) R.vm.kill_object(alloc24) R.vm.kill_object(alloc27) R.vm.kill_object(model_layers_2_post_attention_layernorm_weight4) lv8: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc28, alloc29 model_layers_2_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[15] gv48: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc30: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv48, R.dtype("float16")) cls.gemma_q0f16_NT_matmul2(alloc28, model_layers_2_mlp_gate_up_proj_weight4, alloc30) R.vm.kill_object(alloc28) R.vm.kill_object(model_layers_2_mlp_gate_up_proj_weight4) gv49: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc31: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv49, R.dtype("float16")) cls.gemma_q0f16_fused_split_gelu_tanh_multiply1(alloc30, alloc31) R.vm.kill_object(alloc30) model_layers_2_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[16] gv50: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc32: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv50, R.dtype("float16")) cls.gemma_q0f16_NT_matmul3(alloc31, model_layers_2_mlp_down_proj_weight4, alloc32) R.vm.kill_object(alloc31) R.vm.kill_object(model_layers_2_mlp_down_proj_weight4) model_layers_3_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[23] gv51: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc33: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv51, R.dtype("float16")) gv52: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc34: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv52, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc32, alloc29, model_layers_3_input_layernorm_weight4, alloc33, alloc34) R.vm.kill_object(alloc29) R.vm.kill_object(alloc32) R.vm.kill_object(model_layers_3_input_layernorm_weight4) lv10: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc33, alloc34 model_layers_3_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[19] gv53: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc35: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv53, R.dtype("float16")) cls.gemma_q0f16_NT_matmul(alloc33, model_layers_3_self_attn_qkv_proj_weight4, alloc35) R.vm.kill_object(alloc33) R.vm.kill_object(model_layers_3_self_attn_qkv_proj_weight4) gv54: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape228: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc35, gv54, sinfo_args=(R.Tensor((batch_size, 1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc35) gv55: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape229: R.Tensor((batch_size, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape228, gv55, sinfo_args=(R.Tensor((batch_size, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape228) gv56: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc36: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv56, R.dtype("float16")) _29: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape229, alloc36) R.vm.kill_object(reshape229) gv57: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape230: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc36, gv57, sinfo_args=(R.Tensor((batch_size, 1, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc36) gv58: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape231: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape230, gv58, sinfo_args=(R.Tensor((batch_size, 1, 2048), dtype="float16"),)) R.vm.kill_object(reshape230) model_layers_3_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[20] gv59: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc37: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv59, R.dtype("float16")) cls.gemma_q0f16_NT_matmul1(reshape231, model_layers_3_self_attn_o_proj_weight4, alloc37) R.vm.kill_object(reshape231) R.vm.kill_object(model_layers_3_self_attn_o_proj_weight4) model_layers_3_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[24] gv60: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc38: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv60, R.dtype("float16")) gv61: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc39: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv61, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc37, alloc34, model_layers_3_post_attention_layernorm_weight4, alloc38, alloc39) R.vm.kill_object(alloc34) R.vm.kill_object(alloc37) R.vm.kill_object(model_layers_3_post_attention_layernorm_weight4) lv12: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc38, alloc39 model_layers_3_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[21] gv62: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc40: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv62, R.dtype("float16")) cls.gemma_q0f16_NT_matmul2(alloc38, model_layers_3_mlp_gate_up_proj_weight4, alloc40) R.vm.kill_object(alloc38) R.vm.kill_object(model_layers_3_mlp_gate_up_proj_weight4) gv63: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc41: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv63, R.dtype("float16")) cls.gemma_q0f16_fused_split_gelu_tanh_multiply1(alloc40, alloc41) R.vm.kill_object(alloc40) model_layers_3_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[22] gv64: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc42: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv64, R.dtype("float16")) cls.gemma_q0f16_NT_matmul3(alloc41, model_layers_3_mlp_down_proj_weight4, alloc42) R.vm.kill_object(alloc41) R.vm.kill_object(model_layers_3_mlp_down_proj_weight4) model_layers_4_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[29] gv65: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc43: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv65, R.dtype("float16")) gv66: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc44: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv66, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc42, alloc39, model_layers_4_input_layernorm_weight4, alloc43, alloc44) R.vm.kill_object(alloc39) R.vm.kill_object(alloc42) R.vm.kill_object(model_layers_4_input_layernorm_weight4) lv14: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc43, alloc44 model_layers_4_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[25] gv67: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc45: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv67, R.dtype("float16")) cls.gemma_q0f16_NT_matmul(alloc43, model_layers_4_self_attn_qkv_proj_weight4, alloc45) R.vm.kill_object(alloc43) R.vm.kill_object(model_layers_4_self_attn_qkv_proj_weight4) gv68: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape232: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc45, gv68, sinfo_args=(R.Tensor((batch_size, 1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc45) gv69: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape233: R.Tensor((batch_size, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape232, gv69, sinfo_args=(R.Tensor((batch_size, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape232) gv70: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc46: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv70, R.dtype("float16")) _37: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape233, alloc46) R.vm.kill_object(reshape233) gv71: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape234: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc46, gv71, sinfo_args=(R.Tensor((batch_size, 1, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc46) gv72: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape235: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape234, gv72, sinfo_args=(R.Tensor((batch_size, 1, 2048), dtype="float16"),)) R.vm.kill_object(reshape234) model_layers_4_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[26] gv73: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc47: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv73, R.dtype("float16")) cls.gemma_q0f16_NT_matmul1(reshape235, model_layers_4_self_attn_o_proj_weight4, alloc47) R.vm.kill_object(reshape235) R.vm.kill_object(model_layers_4_self_attn_o_proj_weight4) model_layers_4_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[30] gv74: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc48: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv74, R.dtype("float16")) gv75: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc49: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv75, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc47, alloc44, model_layers_4_post_attention_layernorm_weight4, alloc48, alloc49) R.vm.kill_object(alloc44) R.vm.kill_object(alloc47) R.vm.kill_object(model_layers_4_post_attention_layernorm_weight4) lv16: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc48, alloc49 model_layers_4_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[27] gv76: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc50: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv76, R.dtype("float16")) cls.gemma_q0f16_NT_matmul2(alloc48, model_layers_4_mlp_gate_up_proj_weight4, alloc50) R.vm.kill_object(alloc48) R.vm.kill_object(model_layers_4_mlp_gate_up_proj_weight4) gv77: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc51: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv77, R.dtype("float16")) cls.gemma_q0f16_fused_split_gelu_tanh_multiply1(alloc50, alloc51) R.vm.kill_object(alloc50) model_layers_4_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[28] gv78: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc52: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv78, R.dtype("float16")) cls.gemma_q0f16_NT_matmul3(alloc51, model_layers_4_mlp_down_proj_weight4, alloc52) R.vm.kill_object(alloc51) R.vm.kill_object(model_layers_4_mlp_down_proj_weight4) model_layers_5_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[35] gv79: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc53: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv79, R.dtype("float16")) gv80: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc54: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv80, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc52, alloc49, model_layers_5_input_layernorm_weight4, alloc53, alloc54) R.vm.kill_object(alloc49) R.vm.kill_object(alloc52) R.vm.kill_object(model_layers_5_input_layernorm_weight4) lv18: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc53, alloc54 model_layers_5_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[31] gv81: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc55: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv81, R.dtype("float16")) cls.gemma_q0f16_NT_matmul(alloc53, model_layers_5_self_attn_qkv_proj_weight4, alloc55) R.vm.kill_object(alloc53) R.vm.kill_object(model_layers_5_self_attn_qkv_proj_weight4) gv82: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape236: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc55, gv82, sinfo_args=(R.Tensor((batch_size, 1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc55) gv83: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape237: R.Tensor((batch_size, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape236, gv83, sinfo_args=(R.Tensor((batch_size, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape236) gv84: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc56: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv84, R.dtype("float16")) _45: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape237, alloc56) R.vm.kill_object(reshape237) gv85: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape238: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc56, gv85, sinfo_args=(R.Tensor((batch_size, 1, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc56) gv86: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape239: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape238, gv86, sinfo_args=(R.Tensor((batch_size, 1, 2048), dtype="float16"),)) R.vm.kill_object(reshape238) model_layers_5_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[32] gv87: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc57: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv87, R.dtype("float16")) cls.gemma_q0f16_NT_matmul1(reshape239, model_layers_5_self_attn_o_proj_weight4, alloc57) R.vm.kill_object(reshape239) R.vm.kill_object(model_layers_5_self_attn_o_proj_weight4) model_layers_5_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[36] gv88: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc58: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv88, R.dtype("float16")) gv89: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc59: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv89, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc57, alloc54, model_layers_5_post_attention_layernorm_weight4, alloc58, alloc59) R.vm.kill_object(alloc54) R.vm.kill_object(alloc57) R.vm.kill_object(model_layers_5_post_attention_layernorm_weight4) lv20: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc58, alloc59 model_layers_5_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[33] gv90: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc60: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv90, R.dtype("float16")) cls.gemma_q0f16_NT_matmul2(alloc58, model_layers_5_mlp_gate_up_proj_weight4, alloc60) R.vm.kill_object(alloc58) R.vm.kill_object(model_layers_5_mlp_gate_up_proj_weight4) gv91: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc61: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv91, R.dtype("float16")) cls.gemma_q0f16_fused_split_gelu_tanh_multiply1(alloc60, alloc61) R.vm.kill_object(alloc60) model_layers_5_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[34] gv92: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc62: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv92, R.dtype("float16")) cls.gemma_q0f16_NT_matmul3(alloc61, model_layers_5_mlp_down_proj_weight4, alloc62) R.vm.kill_object(alloc61) R.vm.kill_object(model_layers_5_mlp_down_proj_weight4) model_layers_6_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[41] gv93: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc63: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv93, R.dtype("float16")) gv94: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc64: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv94, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc62, alloc59, model_layers_6_input_layernorm_weight4, alloc63, alloc64) R.vm.kill_object(alloc59) R.vm.kill_object(alloc62) R.vm.kill_object(model_layers_6_input_layernorm_weight4) lv22: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc63, alloc64 model_layers_6_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[37] gv95: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc65: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv95, R.dtype("float16")) cls.gemma_q0f16_NT_matmul(alloc63, model_layers_6_self_attn_qkv_proj_weight4, alloc65) R.vm.kill_object(alloc63) R.vm.kill_object(model_layers_6_self_attn_qkv_proj_weight4) gv96: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape240: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc65, gv96, sinfo_args=(R.Tensor((batch_size, 1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc65) gv97: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape241: R.Tensor((batch_size, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape240, gv97, sinfo_args=(R.Tensor((batch_size, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape240) gv98: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc66: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv98, R.dtype("float16")) _53: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(6), R.prim_value(T.float32(1.0)), reshape241, alloc66) R.vm.kill_object(reshape241) gv99: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape242: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc66, gv99, sinfo_args=(R.Tensor((batch_size, 1, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc66) gv100: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape243: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape242, gv100, sinfo_args=(R.Tensor((batch_size, 1, 2048), dtype="float16"),)) R.vm.kill_object(reshape242) model_layers_6_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[38] gv101: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc67: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv101, R.dtype("float16")) cls.gemma_q0f16_NT_matmul1(reshape243, model_layers_6_self_attn_o_proj_weight4, alloc67) R.vm.kill_object(reshape243) R.vm.kill_object(model_layers_6_self_attn_o_proj_weight4) model_layers_6_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[42] gv102: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc68: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv102, R.dtype("float16")) gv103: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc69: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv103, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc67, alloc64, model_layers_6_post_attention_layernorm_weight4, alloc68, alloc69) R.vm.kill_object(alloc64) R.vm.kill_object(alloc67) R.vm.kill_object(model_layers_6_post_attention_layernorm_weight4) lv24: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc68, alloc69 model_layers_6_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[39] gv104: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc70: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv104, R.dtype("float16")) cls.gemma_q0f16_NT_matmul2(alloc68, model_layers_6_mlp_gate_up_proj_weight4, alloc70) R.vm.kill_object(alloc68) R.vm.kill_object(model_layers_6_mlp_gate_up_proj_weight4) gv105: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc71: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv105, R.dtype("float16")) cls.gemma_q0f16_fused_split_gelu_tanh_multiply1(alloc70, alloc71) R.vm.kill_object(alloc70) model_layers_6_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[40] gv106: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc72: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv106, R.dtype("float16")) cls.gemma_q0f16_NT_matmul3(alloc71, model_layers_6_mlp_down_proj_weight4, alloc72) R.vm.kill_object(alloc71) R.vm.kill_object(model_layers_6_mlp_down_proj_weight4) model_layers_7_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[47] gv107: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc73: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv107, R.dtype("float16")) gv108: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc74: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv108, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc72, alloc69, model_layers_7_input_layernorm_weight4, alloc73, alloc74) R.vm.kill_object(alloc69) R.vm.kill_object(alloc72) R.vm.kill_object(model_layers_7_input_layernorm_weight4) lv26: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc73, alloc74 model_layers_7_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[43] gv109: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc75: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv109, R.dtype("float16")) cls.gemma_q0f16_NT_matmul(alloc73, model_layers_7_self_attn_qkv_proj_weight4, alloc75) R.vm.kill_object(alloc73) R.vm.kill_object(model_layers_7_self_attn_qkv_proj_weight4) gv110: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape244: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc75, gv110, sinfo_args=(R.Tensor((batch_size, 1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc75) gv111: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape245: R.Tensor((batch_size, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape244, gv111, sinfo_args=(R.Tensor((batch_size, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape244) gv112: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc76: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv112, R.dtype("float16")) _61: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(7), R.prim_value(T.float32(1.0)), reshape245, alloc76) R.vm.kill_object(reshape245) gv113: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape246: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc76, gv113, sinfo_args=(R.Tensor((batch_size, 1, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc76) gv114: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape247: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape246, gv114, sinfo_args=(R.Tensor((batch_size, 1, 2048), dtype="float16"),)) R.vm.kill_object(reshape246) model_layers_7_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[44] gv115: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc77: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv115, R.dtype("float16")) cls.gemma_q0f16_NT_matmul1(reshape247, model_layers_7_self_attn_o_proj_weight4, alloc77) R.vm.kill_object(reshape247) R.vm.kill_object(model_layers_7_self_attn_o_proj_weight4) model_layers_7_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[48] gv116: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc78: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv116, R.dtype("float16")) gv117: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc79: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv117, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc77, alloc74, model_layers_7_post_attention_layernorm_weight4, alloc78, alloc79) R.vm.kill_object(alloc74) R.vm.kill_object(alloc77) R.vm.kill_object(model_layers_7_post_attention_layernorm_weight4) lv28: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc78, alloc79 model_layers_7_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[45] gv118: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc80: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv118, R.dtype("float16")) cls.gemma_q0f16_NT_matmul2(alloc78, model_layers_7_mlp_gate_up_proj_weight4, alloc80) R.vm.kill_object(alloc78) R.vm.kill_object(model_layers_7_mlp_gate_up_proj_weight4) gv119: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc81: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv119, R.dtype("float16")) cls.gemma_q0f16_fused_split_gelu_tanh_multiply1(alloc80, alloc81) R.vm.kill_object(alloc80) model_layers_7_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[46] gv120: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc82: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv120, R.dtype("float16")) cls.gemma_q0f16_NT_matmul3(alloc81, model_layers_7_mlp_down_proj_weight4, alloc82) R.vm.kill_object(alloc81) R.vm.kill_object(model_layers_7_mlp_down_proj_weight4) model_layers_8_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[53] gv121: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc83: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv121, R.dtype("float16")) gv122: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc84: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv122, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc82, alloc79, model_layers_8_input_layernorm_weight4, alloc83, alloc84) R.vm.kill_object(alloc79) R.vm.kill_object(alloc82) R.vm.kill_object(model_layers_8_input_layernorm_weight4) lv30: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc83, alloc84 model_layers_8_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[49] gv123: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc85: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv123, R.dtype("float16")) cls.gemma_q0f16_NT_matmul(alloc83, model_layers_8_self_attn_qkv_proj_weight4, alloc85) R.vm.kill_object(alloc83) R.vm.kill_object(model_layers_8_self_attn_qkv_proj_weight4) gv124: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape248: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc85, gv124, sinfo_args=(R.Tensor((batch_size, 1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc85) gv125: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape249: R.Tensor((batch_size, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape248, gv125, sinfo_args=(R.Tensor((batch_size, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape248) gv126: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc86: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv126, R.dtype("float16")) _69: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(8), R.prim_value(T.float32(1.0)), reshape249, alloc86) R.vm.kill_object(reshape249) gv127: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape250: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc86, gv127, sinfo_args=(R.Tensor((batch_size, 1, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc86) gv128: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape251: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape250, gv128, sinfo_args=(R.Tensor((batch_size, 1, 2048), dtype="float16"),)) R.vm.kill_object(reshape250) model_layers_8_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[50] gv129: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc87: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv129, R.dtype("float16")) cls.gemma_q0f16_NT_matmul1(reshape251, model_layers_8_self_attn_o_proj_weight4, alloc87) R.vm.kill_object(reshape251) R.vm.kill_object(model_layers_8_self_attn_o_proj_weight4) model_layers_8_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[54] gv130: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc88: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv130, R.dtype("float16")) gv131: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc89: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv131, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc87, alloc84, model_layers_8_post_attention_layernorm_weight4, alloc88, alloc89) R.vm.kill_object(alloc84) R.vm.kill_object(alloc87) R.vm.kill_object(model_layers_8_post_attention_layernorm_weight4) lv32: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc88, alloc89 model_layers_8_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[51] gv132: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc90: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv132, R.dtype("float16")) cls.gemma_q0f16_NT_matmul2(alloc88, model_layers_8_mlp_gate_up_proj_weight4, alloc90) R.vm.kill_object(alloc88) R.vm.kill_object(model_layers_8_mlp_gate_up_proj_weight4) gv133: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc91: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv133, R.dtype("float16")) cls.gemma_q0f16_fused_split_gelu_tanh_multiply1(alloc90, alloc91) R.vm.kill_object(alloc90) model_layers_8_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[52] gv134: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc92: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv134, R.dtype("float16")) cls.gemma_q0f16_NT_matmul3(alloc91, model_layers_8_mlp_down_proj_weight4, alloc92) R.vm.kill_object(alloc91) R.vm.kill_object(model_layers_8_mlp_down_proj_weight4) model_layers_9_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[59] gv135: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc93: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv135, R.dtype("float16")) gv136: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc94: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv136, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc92, alloc89, model_layers_9_input_layernorm_weight4, alloc93, alloc94) R.vm.kill_object(alloc89) R.vm.kill_object(alloc92) R.vm.kill_object(model_layers_9_input_layernorm_weight4) lv34: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc93, alloc94 model_layers_9_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[55] gv137: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc95: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv137, R.dtype("float16")) cls.gemma_q0f16_NT_matmul(alloc93, model_layers_9_self_attn_qkv_proj_weight4, alloc95) R.vm.kill_object(alloc93) R.vm.kill_object(model_layers_9_self_attn_qkv_proj_weight4) gv138: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape252: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc95, gv138, sinfo_args=(R.Tensor((batch_size, 1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc95) gv139: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape253: R.Tensor((batch_size, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape252, gv139, sinfo_args=(R.Tensor((batch_size, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape252) gv140: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc96: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv140, R.dtype("float16")) _77: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(9), R.prim_value(T.float32(1.0)), reshape253, alloc96) R.vm.kill_object(reshape253) gv141: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape254: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc96, gv141, sinfo_args=(R.Tensor((batch_size, 1, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc96) gv142: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape255: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape254, gv142, sinfo_args=(R.Tensor((batch_size, 1, 2048), dtype="float16"),)) R.vm.kill_object(reshape254) model_layers_9_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[56] gv143: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc97: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv143, R.dtype("float16")) cls.gemma_q0f16_NT_matmul1(reshape255, model_layers_9_self_attn_o_proj_weight4, alloc97) R.vm.kill_object(reshape255) R.vm.kill_object(model_layers_9_self_attn_o_proj_weight4) model_layers_9_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[60] gv144: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc98: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv144, R.dtype("float16")) gv145: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc99: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv145, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc97, alloc94, model_layers_9_post_attention_layernorm_weight4, alloc98, alloc99) R.vm.kill_object(alloc94) R.vm.kill_object(alloc97) R.vm.kill_object(model_layers_9_post_attention_layernorm_weight4) lv36: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc98, alloc99 model_layers_9_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[57] gv146: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc100: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv146, R.dtype("float16")) cls.gemma_q0f16_NT_matmul2(alloc98, model_layers_9_mlp_gate_up_proj_weight4, alloc100) R.vm.kill_object(alloc98) R.vm.kill_object(model_layers_9_mlp_gate_up_proj_weight4) gv147: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc101: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv147, R.dtype("float16")) cls.gemma_q0f16_fused_split_gelu_tanh_multiply1(alloc100, alloc101) R.vm.kill_object(alloc100) model_layers_9_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[58] gv148: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc102: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv148, R.dtype("float16")) cls.gemma_q0f16_NT_matmul3(alloc101, model_layers_9_mlp_down_proj_weight4, alloc102) R.vm.kill_object(alloc101) R.vm.kill_object(model_layers_9_mlp_down_proj_weight4) model_layers_10_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[65] gv149: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc103: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv149, R.dtype("float16")) gv150: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc104: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv150, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc102, alloc99, model_layers_10_input_layernorm_weight4, alloc103, alloc104) R.vm.kill_object(alloc99) R.vm.kill_object(alloc102) R.vm.kill_object(model_layers_10_input_layernorm_weight4) lv38: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc103, alloc104 model_layers_10_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[61] gv151: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc105: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv151, R.dtype("float16")) cls.gemma_q0f16_NT_matmul(alloc103, model_layers_10_self_attn_qkv_proj_weight4, alloc105) R.vm.kill_object(alloc103) R.vm.kill_object(model_layers_10_self_attn_qkv_proj_weight4) gv152: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape256: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc105, gv152, sinfo_args=(R.Tensor((batch_size, 1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc105) gv153: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape257: R.Tensor((batch_size, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape256, gv153, sinfo_args=(R.Tensor((batch_size, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape256) gv154: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc106: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv154, R.dtype("float16")) _85: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(10), R.prim_value(T.float32(1.0)), reshape257, alloc106) R.vm.kill_object(reshape257) gv155: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape258: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc106, gv155, sinfo_args=(R.Tensor((batch_size, 1, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc106) gv156: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape259: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape258, gv156, sinfo_args=(R.Tensor((batch_size, 1, 2048), dtype="float16"),)) R.vm.kill_object(reshape258) model_layers_10_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[62] gv157: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc107: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv157, R.dtype("float16")) cls.gemma_q0f16_NT_matmul1(reshape259, model_layers_10_self_attn_o_proj_weight4, alloc107) R.vm.kill_object(reshape259) R.vm.kill_object(model_layers_10_self_attn_o_proj_weight4) model_layers_10_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[66] gv158: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc108: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv158, R.dtype("float16")) gv159: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc109: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv159, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc107, alloc104, model_layers_10_post_attention_layernorm_weight4, alloc108, alloc109) R.vm.kill_object(alloc104) R.vm.kill_object(alloc107) R.vm.kill_object(model_layers_10_post_attention_layernorm_weight4) lv40: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc108, alloc109 model_layers_10_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[63] gv160: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc110: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv160, R.dtype("float16")) cls.gemma_q0f16_NT_matmul2(alloc108, model_layers_10_mlp_gate_up_proj_weight4, alloc110) R.vm.kill_object(alloc108) R.vm.kill_object(model_layers_10_mlp_gate_up_proj_weight4) gv161: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc111: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv161, R.dtype("float16")) cls.gemma_q0f16_fused_split_gelu_tanh_multiply1(alloc110, alloc111) R.vm.kill_object(alloc110) model_layers_10_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[64] gv162: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc112: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv162, R.dtype("float16")) cls.gemma_q0f16_NT_matmul3(alloc111, model_layers_10_mlp_down_proj_weight4, alloc112) R.vm.kill_object(alloc111) R.vm.kill_object(model_layers_10_mlp_down_proj_weight4) model_layers_11_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[71] gv163: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc113: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv163, R.dtype("float16")) gv164: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc114: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv164, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc112, alloc109, model_layers_11_input_layernorm_weight4, alloc113, alloc114) R.vm.kill_object(alloc109) R.vm.kill_object(alloc112) R.vm.kill_object(model_layers_11_input_layernorm_weight4) lv42: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc113, alloc114 model_layers_11_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[67] gv165: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc115: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv165, R.dtype("float16")) cls.gemma_q0f16_NT_matmul(alloc113, model_layers_11_self_attn_qkv_proj_weight4, alloc115) R.vm.kill_object(alloc113) R.vm.kill_object(model_layers_11_self_attn_qkv_proj_weight4) gv166: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape260: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc115, gv166, sinfo_args=(R.Tensor((batch_size, 1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc115) gv167: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape261: R.Tensor((batch_size, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape260, gv167, sinfo_args=(R.Tensor((batch_size, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape260) gv168: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc116: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv168, R.dtype("float16")) _93: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(11), R.prim_value(T.float32(1.0)), reshape261, alloc116) R.vm.kill_object(reshape261) gv169: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape262: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc116, gv169, sinfo_args=(R.Tensor((batch_size, 1, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc116) gv170: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape263: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape262, gv170, sinfo_args=(R.Tensor((batch_size, 1, 2048), dtype="float16"),)) R.vm.kill_object(reshape262) model_layers_11_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[68] gv171: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc117: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv171, R.dtype("float16")) cls.gemma_q0f16_NT_matmul1(reshape263, model_layers_11_self_attn_o_proj_weight4, alloc117) R.vm.kill_object(reshape263) R.vm.kill_object(model_layers_11_self_attn_o_proj_weight4) model_layers_11_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[72] gv172: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc118: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv172, R.dtype("float16")) gv173: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc119: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv173, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc117, alloc114, model_layers_11_post_attention_layernorm_weight4, alloc118, alloc119) R.vm.kill_object(alloc114) R.vm.kill_object(alloc117) R.vm.kill_object(model_layers_11_post_attention_layernorm_weight4) lv44: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc118, alloc119 model_layers_11_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[69] gv174: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc120: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv174, R.dtype("float16")) cls.gemma_q0f16_NT_matmul2(alloc118, model_layers_11_mlp_gate_up_proj_weight4, alloc120) R.vm.kill_object(alloc118) R.vm.kill_object(model_layers_11_mlp_gate_up_proj_weight4) gv175: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc121: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv175, R.dtype("float16")) cls.gemma_q0f16_fused_split_gelu_tanh_multiply1(alloc120, alloc121) R.vm.kill_object(alloc120) model_layers_11_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[70] gv176: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc122: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv176, R.dtype("float16")) cls.gemma_q0f16_NT_matmul3(alloc121, model_layers_11_mlp_down_proj_weight4, alloc122) R.vm.kill_object(alloc121) R.vm.kill_object(model_layers_11_mlp_down_proj_weight4) model_layers_12_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[77] gv177: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc123: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv177, R.dtype("float16")) gv178: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc124: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv178, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc122, alloc119, model_layers_12_input_layernorm_weight4, alloc123, alloc124) R.vm.kill_object(alloc119) R.vm.kill_object(alloc122) R.vm.kill_object(model_layers_12_input_layernorm_weight4) lv46: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc123, alloc124 model_layers_12_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[73] gv179: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc125: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv179, R.dtype("float16")) cls.gemma_q0f16_NT_matmul(alloc123, model_layers_12_self_attn_qkv_proj_weight4, alloc125) R.vm.kill_object(alloc123) R.vm.kill_object(model_layers_12_self_attn_qkv_proj_weight4) gv180: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape264: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc125, gv180, sinfo_args=(R.Tensor((batch_size, 1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc125) gv181: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape265: R.Tensor((batch_size, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape264, gv181, sinfo_args=(R.Tensor((batch_size, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape264) gv182: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc126: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv182, R.dtype("float16")) _101: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(12), R.prim_value(T.float32(1.0)), reshape265, alloc126) R.vm.kill_object(reshape265) gv183: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape266: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc126, gv183, sinfo_args=(R.Tensor((batch_size, 1, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc126) gv184: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape267: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape266, gv184, sinfo_args=(R.Tensor((batch_size, 1, 2048), dtype="float16"),)) R.vm.kill_object(reshape266) model_layers_12_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[74] gv185: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc127: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv185, R.dtype("float16")) cls.gemma_q0f16_NT_matmul1(reshape267, model_layers_12_self_attn_o_proj_weight4, alloc127) R.vm.kill_object(reshape267) R.vm.kill_object(model_layers_12_self_attn_o_proj_weight4) model_layers_12_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[78] gv186: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc128: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv186, R.dtype("float16")) gv187: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc129: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv187, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc127, alloc124, model_layers_12_post_attention_layernorm_weight4, alloc128, alloc129) R.vm.kill_object(alloc124) R.vm.kill_object(alloc127) R.vm.kill_object(model_layers_12_post_attention_layernorm_weight4) lv48: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc128, alloc129 model_layers_12_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[75] gv188: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc130: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv188, R.dtype("float16")) cls.gemma_q0f16_NT_matmul2(alloc128, model_layers_12_mlp_gate_up_proj_weight4, alloc130) R.vm.kill_object(alloc128) R.vm.kill_object(model_layers_12_mlp_gate_up_proj_weight4) gv189: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc131: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv189, R.dtype("float16")) cls.gemma_q0f16_fused_split_gelu_tanh_multiply1(alloc130, alloc131) R.vm.kill_object(alloc130) model_layers_12_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[76] gv190: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc132: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv190, R.dtype("float16")) cls.gemma_q0f16_NT_matmul3(alloc131, model_layers_12_mlp_down_proj_weight4, alloc132) R.vm.kill_object(alloc131) R.vm.kill_object(model_layers_12_mlp_down_proj_weight4) model_layers_13_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[83] gv191: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc133: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv191, R.dtype("float16")) gv192: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc134: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv192, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc132, alloc129, model_layers_13_input_layernorm_weight4, alloc133, alloc134) R.vm.kill_object(alloc129) R.vm.kill_object(alloc132) R.vm.kill_object(model_layers_13_input_layernorm_weight4) lv50: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc133, alloc134 model_layers_13_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[79] gv193: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc135: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv193, R.dtype("float16")) cls.gemma_q0f16_NT_matmul(alloc133, model_layers_13_self_attn_qkv_proj_weight4, alloc135) R.vm.kill_object(alloc133) R.vm.kill_object(model_layers_13_self_attn_qkv_proj_weight4) gv194: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape268: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc135, gv194, sinfo_args=(R.Tensor((batch_size, 1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc135) gv195: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape269: R.Tensor((batch_size, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape268, gv195, sinfo_args=(R.Tensor((batch_size, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape268) gv196: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc136: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv196, R.dtype("float16")) _109: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(13), R.prim_value(T.float32(1.0)), reshape269, alloc136) R.vm.kill_object(reshape269) gv197: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape270: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc136, gv197, sinfo_args=(R.Tensor((batch_size, 1, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc136) gv198: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape271: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape270, gv198, sinfo_args=(R.Tensor((batch_size, 1, 2048), dtype="float16"),)) R.vm.kill_object(reshape270) model_layers_13_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[80] gv199: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc137: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv199, R.dtype("float16")) cls.gemma_q0f16_NT_matmul1(reshape271, model_layers_13_self_attn_o_proj_weight4, alloc137) R.vm.kill_object(reshape271) R.vm.kill_object(model_layers_13_self_attn_o_proj_weight4) model_layers_13_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[84] gv200: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc138: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv200, R.dtype("float16")) gv201: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc139: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv201, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc137, alloc134, model_layers_13_post_attention_layernorm_weight4, alloc138, alloc139) R.vm.kill_object(alloc134) R.vm.kill_object(alloc137) R.vm.kill_object(model_layers_13_post_attention_layernorm_weight4) lv52: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc138, alloc139 model_layers_13_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[81] gv202: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc140: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv202, R.dtype("float16")) cls.gemma_q0f16_NT_matmul2(alloc138, model_layers_13_mlp_gate_up_proj_weight4, alloc140) R.vm.kill_object(alloc138) R.vm.kill_object(model_layers_13_mlp_gate_up_proj_weight4) gv203: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc141: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv203, R.dtype("float16")) cls.gemma_q0f16_fused_split_gelu_tanh_multiply1(alloc140, alloc141) R.vm.kill_object(alloc140) model_layers_13_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[82] gv204: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc142: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv204, R.dtype("float16")) cls.gemma_q0f16_NT_matmul3(alloc141, model_layers_13_mlp_down_proj_weight4, alloc142) R.vm.kill_object(alloc141) R.vm.kill_object(model_layers_13_mlp_down_proj_weight4) model_layers_14_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[89] gv205: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc143: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv205, R.dtype("float16")) gv206: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc144: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv206, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc142, alloc139, model_layers_14_input_layernorm_weight4, alloc143, alloc144) R.vm.kill_object(alloc139) R.vm.kill_object(alloc142) R.vm.kill_object(model_layers_14_input_layernorm_weight4) lv54: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc143, alloc144 model_layers_14_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[85] gv207: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc145: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv207, R.dtype("float16")) cls.gemma_q0f16_NT_matmul(alloc143, model_layers_14_self_attn_qkv_proj_weight4, alloc145) R.vm.kill_object(alloc143) R.vm.kill_object(model_layers_14_self_attn_qkv_proj_weight4) gv208: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape272: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc145, gv208, sinfo_args=(R.Tensor((batch_size, 1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc145) gv209: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape273: R.Tensor((batch_size, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape272, gv209, sinfo_args=(R.Tensor((batch_size, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape272) gv210: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc146: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv210, R.dtype("float16")) _117: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(14), R.prim_value(T.float32(1.0)), reshape273, alloc146) R.vm.kill_object(reshape273) gv211: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape274: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc146, gv211, sinfo_args=(R.Tensor((batch_size, 1, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc146) gv212: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape275: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape274, gv212, sinfo_args=(R.Tensor((batch_size, 1, 2048), dtype="float16"),)) R.vm.kill_object(reshape274) model_layers_14_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[86] gv213: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc147: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv213, R.dtype("float16")) cls.gemma_q0f16_NT_matmul1(reshape275, model_layers_14_self_attn_o_proj_weight4, alloc147) R.vm.kill_object(reshape275) R.vm.kill_object(model_layers_14_self_attn_o_proj_weight4) model_layers_14_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[90] gv214: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc148: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv214, R.dtype("float16")) gv215: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc149: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv215, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc147, alloc144, model_layers_14_post_attention_layernorm_weight4, alloc148, alloc149) R.vm.kill_object(alloc144) R.vm.kill_object(alloc147) R.vm.kill_object(model_layers_14_post_attention_layernorm_weight4) lv56: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc148, alloc149 model_layers_14_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[87] gv216: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc150: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv216, R.dtype("float16")) cls.gemma_q0f16_NT_matmul2(alloc148, model_layers_14_mlp_gate_up_proj_weight4, alloc150) R.vm.kill_object(alloc148) R.vm.kill_object(model_layers_14_mlp_gate_up_proj_weight4) gv217: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc151: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv217, R.dtype("float16")) cls.gemma_q0f16_fused_split_gelu_tanh_multiply1(alloc150, alloc151) R.vm.kill_object(alloc150) model_layers_14_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[88] gv218: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc152: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv218, R.dtype("float16")) cls.gemma_q0f16_NT_matmul3(alloc151, model_layers_14_mlp_down_proj_weight4, alloc152) R.vm.kill_object(alloc151) R.vm.kill_object(model_layers_14_mlp_down_proj_weight4) model_layers_15_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[95] gv219: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc153: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv219, R.dtype("float16")) gv220: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc154: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv220, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc152, alloc149, model_layers_15_input_layernorm_weight4, alloc153, alloc154) R.vm.kill_object(alloc149) R.vm.kill_object(alloc152) R.vm.kill_object(model_layers_15_input_layernorm_weight4) lv58: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc153, alloc154 model_layers_15_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[91] gv221: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc155: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv221, R.dtype("float16")) cls.gemma_q0f16_NT_matmul(alloc153, model_layers_15_self_attn_qkv_proj_weight4, alloc155) R.vm.kill_object(alloc153) R.vm.kill_object(model_layers_15_self_attn_qkv_proj_weight4) gv222: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape276: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc155, gv222, sinfo_args=(R.Tensor((batch_size, 1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc155) gv223: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape277: R.Tensor((batch_size, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape276, gv223, sinfo_args=(R.Tensor((batch_size, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape276) gv224: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc156: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv224, R.dtype("float16")) _125: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(15), R.prim_value(T.float32(1.0)), reshape277, alloc156) R.vm.kill_object(reshape277) gv225: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape278: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc156, gv225, sinfo_args=(R.Tensor((batch_size, 1, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc156) gv226: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape279: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape278, gv226, sinfo_args=(R.Tensor((batch_size, 1, 2048), dtype="float16"),)) R.vm.kill_object(reshape278) model_layers_15_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[92] gv227: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc157: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv227, R.dtype("float16")) cls.gemma_q0f16_NT_matmul1(reshape279, model_layers_15_self_attn_o_proj_weight4, alloc157) R.vm.kill_object(reshape279) R.vm.kill_object(model_layers_15_self_attn_o_proj_weight4) model_layers_15_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[96] gv228: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc158: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv228, R.dtype("float16")) gv229: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc159: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv229, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc157, alloc154, model_layers_15_post_attention_layernorm_weight4, alloc158, alloc159) R.vm.kill_object(alloc154) R.vm.kill_object(alloc157) R.vm.kill_object(model_layers_15_post_attention_layernorm_weight4) lv60: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc158, alloc159 model_layers_15_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[93] gv230: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc160: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv230, R.dtype("float16")) cls.gemma_q0f16_NT_matmul2(alloc158, model_layers_15_mlp_gate_up_proj_weight4, alloc160) R.vm.kill_object(alloc158) R.vm.kill_object(model_layers_15_mlp_gate_up_proj_weight4) gv231: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc161: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv231, R.dtype("float16")) cls.gemma_q0f16_fused_split_gelu_tanh_multiply1(alloc160, alloc161) R.vm.kill_object(alloc160) model_layers_15_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[94] gv232: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc162: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv232, R.dtype("float16")) cls.gemma_q0f16_NT_matmul3(alloc161, model_layers_15_mlp_down_proj_weight4, alloc162) R.vm.kill_object(alloc161) R.vm.kill_object(model_layers_15_mlp_down_proj_weight4) model_layers_16_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[101] gv233: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc163: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv233, R.dtype("float16")) gv234: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc164: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv234, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc162, alloc159, model_layers_16_input_layernorm_weight4, alloc163, alloc164) R.vm.kill_object(alloc159) R.vm.kill_object(alloc162) R.vm.kill_object(model_layers_16_input_layernorm_weight4) lv62: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc163, alloc164 model_layers_16_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[97] gv235: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc165: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv235, R.dtype("float16")) cls.gemma_q0f16_NT_matmul(alloc163, model_layers_16_self_attn_qkv_proj_weight4, alloc165) R.vm.kill_object(alloc163) R.vm.kill_object(model_layers_16_self_attn_qkv_proj_weight4) gv236: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape280: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc165, gv236, sinfo_args=(R.Tensor((batch_size, 1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc165) gv237: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape281: R.Tensor((batch_size, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape280, gv237, sinfo_args=(R.Tensor((batch_size, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape280) gv238: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc166: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv238, R.dtype("float16")) _133: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(16), R.prim_value(T.float32(1.0)), reshape281, alloc166) R.vm.kill_object(reshape281) gv239: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape282: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc166, gv239, sinfo_args=(R.Tensor((batch_size, 1, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc166) gv240: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape283: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape282, gv240, sinfo_args=(R.Tensor((batch_size, 1, 2048), dtype="float16"),)) R.vm.kill_object(reshape282) model_layers_16_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[98] gv241: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc167: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv241, R.dtype("float16")) cls.gemma_q0f16_NT_matmul1(reshape283, model_layers_16_self_attn_o_proj_weight4, alloc167) R.vm.kill_object(reshape283) R.vm.kill_object(model_layers_16_self_attn_o_proj_weight4) model_layers_16_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[102] gv242: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc168: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv242, R.dtype("float16")) gv243: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc169: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv243, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc167, alloc164, model_layers_16_post_attention_layernorm_weight4, alloc168, alloc169) R.vm.kill_object(alloc164) R.vm.kill_object(alloc167) R.vm.kill_object(model_layers_16_post_attention_layernorm_weight4) lv64: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc168, alloc169 model_layers_16_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[99] gv244: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc170: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv244, R.dtype("float16")) cls.gemma_q0f16_NT_matmul2(alloc168, model_layers_16_mlp_gate_up_proj_weight4, alloc170) R.vm.kill_object(alloc168) R.vm.kill_object(model_layers_16_mlp_gate_up_proj_weight4) gv245: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc171: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv245, R.dtype("float16")) cls.gemma_q0f16_fused_split_gelu_tanh_multiply1(alloc170, alloc171) R.vm.kill_object(alloc170) model_layers_16_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[100] gv246: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc172: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv246, R.dtype("float16")) cls.gemma_q0f16_NT_matmul3(alloc171, model_layers_16_mlp_down_proj_weight4, alloc172) R.vm.kill_object(alloc171) R.vm.kill_object(model_layers_16_mlp_down_proj_weight4) model_layers_17_input_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[107] gv247: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc173: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv247, R.dtype("float16")) gv248: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc174: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv248, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_decode(alloc172, alloc169, model_layers_17_input_layernorm_weight4, alloc173, alloc174) R.vm.kill_object(alloc169) R.vm.kill_object(alloc172) R.vm.kill_object(model_layers_17_input_layernorm_weight4) lv66: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc173, alloc174 model_layers_17_self_attn_qkv_proj_weight4: R.Tensor((2560, 2048), dtype="float16") = packed_params[103] gv249: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc175: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv249, R.dtype("float16")) cls.gemma_q0f16_NT_matmul(alloc173, model_layers_17_self_attn_qkv_proj_weight4, alloc175) R.vm.kill_object(alloc173) R.vm.kill_object(model_layers_17_self_attn_qkv_proj_weight4) gv250: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape284: R.Tensor((batch_size, 1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc175, gv250, sinfo_args=(R.Tensor((batch_size, 1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc175) gv251: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape285: R.Tensor((batch_size, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape284, gv251, sinfo_args=(R.Tensor((batch_size, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape284) gv252: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc176: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv252, R.dtype("float16")) _141: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(17), R.prim_value(T.float32(1.0)), reshape285, alloc176) R.vm.kill_object(reshape285) gv253: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape286: R.Tensor((batch_size, 1, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc176, gv253, sinfo_args=(R.Tensor((batch_size, 1, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc176) gv254: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape287: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape286, gv254, sinfo_args=(R.Tensor((batch_size, 1, 2048), dtype="float16"),)) R.vm.kill_object(reshape286) model_layers_17_self_attn_o_proj_weight4: R.Tensor((2048, 2048), dtype="float16") = packed_params[104] gv255: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc177: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv255, R.dtype("float16")) cls.gemma_q0f16_NT_matmul1(reshape287, model_layers_17_self_attn_o_proj_weight4, alloc177) R.vm.kill_object(reshape287) R.vm.kill_object(model_layers_17_self_attn_o_proj_weight4) model_layers_17_post_attention_layernorm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[108] gv256: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc178: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv256, R.dtype("float16")) gv257: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc179: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage4, R.prim_value(0), gv257, R.dtype("float16")) R.vm.kill_object(storage4) cls.gemma_q0f16_fuse_add_norm_decode(alloc177, alloc174, model_layers_17_post_attention_layernorm_weight4, alloc178, alloc179) R.vm.kill_object(alloc174) R.vm.kill_object(alloc177) R.vm.kill_object(model_layers_17_post_attention_layernorm_weight4) lv68: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc178, alloc179 model_layers_17_mlp_gate_up_proj_weight4: R.Tensor((32768, 2048), dtype="float16") = packed_params[105] gv258: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc180: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv258, R.dtype("float16")) cls.gemma_q0f16_NT_matmul2(alloc178, model_layers_17_mlp_gate_up_proj_weight4, alloc180) R.vm.kill_object(alloc178) R.vm.kill_object(model_layers_17_mlp_gate_up_proj_weight4) gv259: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc181: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv259, R.dtype("float16")) cls.gemma_q0f16_fused_split_gelu_tanh_multiply1(alloc180, alloc181) R.vm.kill_object(alloc180) model_layers_17_mlp_down_proj_weight4: R.Tensor((2048, 16384), dtype="float16") = packed_params[106] gv260: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc182: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage3, R.prim_value(0), gv260, R.dtype("float16")) R.vm.kill_object(storage3) cls.gemma_q0f16_NT_matmul3(alloc181, model_layers_17_mlp_down_proj_weight4, alloc182) R.vm.kill_object(alloc181) R.vm.kill_object(model_layers_17_mlp_down_proj_weight4) model_norm_weight4: R.Tensor((2048,), dtype="float16") = packed_params[109] gv261: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc183: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage6, R.prim_value(0), gv261, R.dtype("float16")) R.vm.kill_object(storage6) gv262: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc184: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage5, R.prim_value(0), gv262, R.dtype("float16")) R.vm.kill_object(storage5) cls.gemma_q0f16_fuse_add_norm_decode(alloc182, alloc179, model_norm_weight4, alloc183, alloc184) R.vm.kill_object(alloc179) R.vm.kill_object(alloc182) R.vm.kill_object(model_norm_weight4) lv70: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc183, alloc184 R.vm.kill_object(alloc184) model_embed_tokens_weight4: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[0] gv263: R.Shape(ndim=1) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(3), sinfo_args=(R.Shape(ndim=1),)) storage7: R.Object = R.vm.alloc_storage(gv263, R.prim_value(0), R.dtype("uint8"), R.str("global")) gv264: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(1), sinfo_args=(R.Shape(ndim=3),)) alloc185: R.Tensor(dtype="float32", ndim=3) = R.vm.alloc_tensor(storage7, R.prim_value(0), gv264, R.dtype("float32")) R.vm.kill_object(storage7) cls.gemma_q0f16_NT_matmul4(alloc183, model_embed_tokens_weight4, alloc185) R.vm.kill_object(alloc183) R.vm.kill_object(model_embed_tokens_weight4) gv4: R.Tuple(R.Tensor(dtype="float32", ndim=3), R.Object) = alloc185, paged_kv_cache R.vm.kill_object(alloc185) gv265: R.Tensor(dtype="float32", ndim=3) = gv4[0] R.call_packed("vm.builtin.match_shape", gv265, shape_heap, R.prim_value(3), R.prim_value(3), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(3), R.prim_value(1), R.str("ErrorContext(fn=batch_decode, loc=return, annotation=R.Tuple(R.Tensor((batch_size, 1, vocab_size), dtype=\"float32\"), R.Object)) "), sinfo_args=(R.Tuple,)) return gv4 @R.function def batch_prefill(input_embeds: R.Tensor((1, "seq_len", 2048), dtype="float16"), logit_positions: R.Tensor(("batch_size",), dtype="int32"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 2048), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"))) -> R.Tuple(R.Tensor((1, "batch_size", "vocab_size"), dtype="float32"), R.Object): batch_size = T.int64() vocab_size = T.int64() seq_len = T.int64() R.func_attr({"num_input": 3, "pipeline_parallel_stages": 1, "relax.force_pure": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 8192, "total_seq_len": 8192}}) cls = Module shape_heap: R.Tensor(dtype="int64", ndim=1) = R.call_builtin_with_ctx("vm.builtin.alloc_shape_heap", (R.prim_value(5),), sinfo_args=(R.Tensor(dtype="int64", ndim=1),)) R.call_packed("vm.builtin.check_tensor_info", input_embeds, R.prim_value(3), R.dtype("float16"), R.str("ErrorContext(fn=batch_prefill, loc=param[0], param=input_embeds, annotation=R.Tensor((1, seq_len, 2048), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_tensor_info", logit_positions, R.prim_value(1), R.dtype("int32"), R.str("ErrorContext(fn=batch_prefill, loc=param[1], param=logit_positions, annotation=R.Tensor((batch_size,), dtype=\"int32\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_tuple_info", packed_params, R.prim_value(110), R.str("ErrorContext(fn=batch_prefill, loc=param[3], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 2048), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv526: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[0] R.call_packed("vm.builtin.check_tensor_info", gv526, R.prim_value(2), R.dtype("float16"), R.str("ErrorContext(fn=batch_prefill, loc=param[3], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 2048), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", input_embeds, shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), R.str("ErrorContext(fn=batch_prefill, loc=param[0], param=input_embeds, annotation=R.Tensor((1, seq_len, 2048), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", logit_positions, shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(1), R.str("ErrorContext(fn=batch_prefill, loc=param[1], param=logit_positions, annotation=R.Tensor((batch_size,), dtype=\"int32\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv526, shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(2), R.prim_value(0), R.prim_value(2048), R.str("ErrorContext(fn=batch_prefill, loc=param[3], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 2048), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) cls.gemma_q0f16_shape_func3(shape_heap) storage15: R.Object = R.vm.alloc_storage(R.shape([33554432]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv527: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc371: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv527, R.dtype("float16")) cls.gemma_q0f16_multiply2(input_embeds, alloc371) model_layers_0_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[5] storage16: R.Object = R.vm.alloc_storage(R.shape([268435456]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv528: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc372: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv528, R.dtype("float16")) cls.gemma_q0f16_rms_norm1(alloc371, model_layers_0_input_layernorm_weight3, alloc372) R.vm.kill_object(model_layers_0_input_layernorm_weight3) model_layers_0_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[1] storage17: R.Object = R.vm.alloc_storage(R.shape([536870912]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv529: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc373: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv529, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc372, model_layers_0_self_attn_qkv_proj_weight3, alloc373) R.vm.kill_object(alloc372) R.vm.kill_object(model_layers_0_self_attn_qkv_proj_weight3) gv530: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape144: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc373, gv530, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc373) gv531: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape145: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape144, gv531, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape144) gv532: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc374: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv532, R.dtype("float16")) _301: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape145, alloc374) R.vm.kill_object(reshape145) gv533: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape146: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc374, gv533, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc374) gv534: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape147: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape146, gv534, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape146) model_layers_0_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[2] gv535: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc375: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv535, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape147, model_layers_0_self_attn_o_proj_weight3, alloc375) R.vm.kill_object(reshape147) R.vm.kill_object(model_layers_0_self_attn_o_proj_weight3) model_layers_0_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[6] gv536: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc376: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv536, R.dtype("float16")) storage18: R.Object = R.vm.alloc_storage(R.shape([268435456]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv537: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc377: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv537, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc375, alloc371, model_layers_0_post_attention_layernorm_weight3, alloc376, alloc377) R.vm.kill_object(alloc371) R.vm.kill_object(alloc375) R.vm.kill_object(model_layers_0_post_attention_layernorm_weight3) lv72: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc376, alloc377 model_layers_0_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[3] gv538: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc378: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv538, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc376, model_layers_0_mlp_gate_up_proj_weight3, alloc378) R.vm.kill_object(alloc376) R.vm.kill_object(model_layers_0_mlp_gate_up_proj_weight3) gv539: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc379: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv539, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc378, alloc379) R.vm.kill_object(alloc378) model_layers_0_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[4] gv540: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc380: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv540, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc379, model_layers_0_mlp_down_proj_weight3, alloc380) R.vm.kill_object(alloc379) R.vm.kill_object(model_layers_0_mlp_down_proj_weight3) model_layers_1_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[11] gv541: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc381: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv541, R.dtype("float16")) gv542: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc382: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv542, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc380, alloc377, model_layers_1_input_layernorm_weight3, alloc381, alloc382) R.vm.kill_object(alloc377) R.vm.kill_object(alloc380) R.vm.kill_object(model_layers_1_input_layernorm_weight3) lv74: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc381, alloc382 model_layers_1_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[7] gv543: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc383: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv543, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc381, model_layers_1_self_attn_qkv_proj_weight3, alloc383) R.vm.kill_object(alloc381) R.vm.kill_object(model_layers_1_self_attn_qkv_proj_weight3) gv544: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape148: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc383, gv544, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc383) gv545: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape149: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape148, gv545, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape148) gv546: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc384: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv546, R.dtype("float16")) _309: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape149, alloc384) R.vm.kill_object(reshape149) gv547: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape150: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc384, gv547, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc384) gv548: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape151: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape150, gv548, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape150) model_layers_1_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[8] gv549: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc385: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv549, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape151, model_layers_1_self_attn_o_proj_weight3, alloc385) R.vm.kill_object(reshape151) R.vm.kill_object(model_layers_1_self_attn_o_proj_weight3) model_layers_1_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[12] gv550: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc386: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv550, R.dtype("float16")) gv551: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc387: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv551, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc385, alloc382, model_layers_1_post_attention_layernorm_weight3, alloc386, alloc387) R.vm.kill_object(alloc382) R.vm.kill_object(alloc385) R.vm.kill_object(model_layers_1_post_attention_layernorm_weight3) lv76: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc386, alloc387 model_layers_1_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[9] gv552: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc388: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv552, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc386, model_layers_1_mlp_gate_up_proj_weight3, alloc388) R.vm.kill_object(alloc386) R.vm.kill_object(model_layers_1_mlp_gate_up_proj_weight3) gv553: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc389: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv553, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc388, alloc389) R.vm.kill_object(alloc388) model_layers_1_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[10] gv554: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc390: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv554, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc389, model_layers_1_mlp_down_proj_weight3, alloc390) R.vm.kill_object(alloc389) R.vm.kill_object(model_layers_1_mlp_down_proj_weight3) model_layers_2_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[17] gv555: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc391: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv555, R.dtype("float16")) gv556: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc392: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv556, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc390, alloc387, model_layers_2_input_layernorm_weight3, alloc391, alloc392) R.vm.kill_object(alloc387) R.vm.kill_object(alloc390) R.vm.kill_object(model_layers_2_input_layernorm_weight3) lv78: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc391, alloc392 model_layers_2_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[13] gv557: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc393: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv557, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc391, model_layers_2_self_attn_qkv_proj_weight3, alloc393) R.vm.kill_object(alloc391) R.vm.kill_object(model_layers_2_self_attn_qkv_proj_weight3) gv558: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape152: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc393, gv558, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc393) gv559: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape153: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape152, gv559, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape152) gv560: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc394: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv560, R.dtype("float16")) _317: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape153, alloc394) R.vm.kill_object(reshape153) gv561: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape154: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc394, gv561, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc394) gv562: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape155: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape154, gv562, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape154) model_layers_2_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[14] gv563: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc395: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv563, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape155, model_layers_2_self_attn_o_proj_weight3, alloc395) R.vm.kill_object(reshape155) R.vm.kill_object(model_layers_2_self_attn_o_proj_weight3) model_layers_2_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[18] gv564: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc396: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv564, R.dtype("float16")) gv565: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc397: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv565, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc395, alloc392, model_layers_2_post_attention_layernorm_weight3, alloc396, alloc397) R.vm.kill_object(alloc392) R.vm.kill_object(alloc395) R.vm.kill_object(model_layers_2_post_attention_layernorm_weight3) lv80: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc396, alloc397 model_layers_2_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[15] gv566: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc398: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv566, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc396, model_layers_2_mlp_gate_up_proj_weight3, alloc398) R.vm.kill_object(alloc396) R.vm.kill_object(model_layers_2_mlp_gate_up_proj_weight3) gv567: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc399: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv567, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc398, alloc399) R.vm.kill_object(alloc398) model_layers_2_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[16] gv568: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc400: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv568, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc399, model_layers_2_mlp_down_proj_weight3, alloc400) R.vm.kill_object(alloc399) R.vm.kill_object(model_layers_2_mlp_down_proj_weight3) model_layers_3_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[23] gv569: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc401: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv569, R.dtype("float16")) gv570: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc402: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv570, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc400, alloc397, model_layers_3_input_layernorm_weight3, alloc401, alloc402) R.vm.kill_object(alloc397) R.vm.kill_object(alloc400) R.vm.kill_object(model_layers_3_input_layernorm_weight3) lv82: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc401, alloc402 model_layers_3_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[19] gv571: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc403: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv571, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc401, model_layers_3_self_attn_qkv_proj_weight3, alloc403) R.vm.kill_object(alloc401) R.vm.kill_object(model_layers_3_self_attn_qkv_proj_weight3) gv572: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape156: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc403, gv572, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc403) gv573: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape157: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape156, gv573, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape156) gv574: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc404: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv574, R.dtype("float16")) _325: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape157, alloc404) R.vm.kill_object(reshape157) gv575: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape158: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc404, gv575, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc404) gv576: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape159: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape158, gv576, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape158) model_layers_3_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[20] gv577: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc405: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv577, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape159, model_layers_3_self_attn_o_proj_weight3, alloc405) R.vm.kill_object(reshape159) R.vm.kill_object(model_layers_3_self_attn_o_proj_weight3) model_layers_3_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[24] gv578: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc406: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv578, R.dtype("float16")) gv579: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc407: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv579, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc405, alloc402, model_layers_3_post_attention_layernorm_weight3, alloc406, alloc407) R.vm.kill_object(alloc402) R.vm.kill_object(alloc405) R.vm.kill_object(model_layers_3_post_attention_layernorm_weight3) lv84: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc406, alloc407 model_layers_3_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[21] gv580: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc408: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv580, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc406, model_layers_3_mlp_gate_up_proj_weight3, alloc408) R.vm.kill_object(alloc406) R.vm.kill_object(model_layers_3_mlp_gate_up_proj_weight3) gv581: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc409: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv581, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc408, alloc409) R.vm.kill_object(alloc408) model_layers_3_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[22] gv582: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc410: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv582, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc409, model_layers_3_mlp_down_proj_weight3, alloc410) R.vm.kill_object(alloc409) R.vm.kill_object(model_layers_3_mlp_down_proj_weight3) model_layers_4_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[29] gv583: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc411: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv583, R.dtype("float16")) gv584: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc412: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv584, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc410, alloc407, model_layers_4_input_layernorm_weight3, alloc411, alloc412) R.vm.kill_object(alloc407) R.vm.kill_object(alloc410) R.vm.kill_object(model_layers_4_input_layernorm_weight3) lv86: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc411, alloc412 model_layers_4_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[25] gv585: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc413: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv585, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc411, model_layers_4_self_attn_qkv_proj_weight3, alloc413) R.vm.kill_object(alloc411) R.vm.kill_object(model_layers_4_self_attn_qkv_proj_weight3) gv586: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape160: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc413, gv586, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc413) gv587: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape161: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape160, gv587, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape160) gv588: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc414: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv588, R.dtype("float16")) _333: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape161, alloc414) R.vm.kill_object(reshape161) gv589: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape162: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc414, gv589, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc414) gv590: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape163: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape162, gv590, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape162) model_layers_4_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[26] gv591: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc415: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv591, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape163, model_layers_4_self_attn_o_proj_weight3, alloc415) R.vm.kill_object(reshape163) R.vm.kill_object(model_layers_4_self_attn_o_proj_weight3) model_layers_4_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[30] gv592: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc416: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv592, R.dtype("float16")) gv593: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc417: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv593, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc415, alloc412, model_layers_4_post_attention_layernorm_weight3, alloc416, alloc417) R.vm.kill_object(alloc412) R.vm.kill_object(alloc415) R.vm.kill_object(model_layers_4_post_attention_layernorm_weight3) lv88: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc416, alloc417 model_layers_4_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[27] gv594: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc418: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv594, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc416, model_layers_4_mlp_gate_up_proj_weight3, alloc418) R.vm.kill_object(alloc416) R.vm.kill_object(model_layers_4_mlp_gate_up_proj_weight3) gv595: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc419: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv595, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc418, alloc419) R.vm.kill_object(alloc418) model_layers_4_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[28] gv596: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc420: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv596, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc419, model_layers_4_mlp_down_proj_weight3, alloc420) R.vm.kill_object(alloc419) R.vm.kill_object(model_layers_4_mlp_down_proj_weight3) model_layers_5_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[35] gv597: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc421: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv597, R.dtype("float16")) gv598: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc422: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv598, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc420, alloc417, model_layers_5_input_layernorm_weight3, alloc421, alloc422) R.vm.kill_object(alloc417) R.vm.kill_object(alloc420) R.vm.kill_object(model_layers_5_input_layernorm_weight3) lv90: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc421, alloc422 model_layers_5_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[31] gv599: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc423: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv599, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc421, model_layers_5_self_attn_qkv_proj_weight3, alloc423) R.vm.kill_object(alloc421) R.vm.kill_object(model_layers_5_self_attn_qkv_proj_weight3) gv600: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape164: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc423, gv600, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc423) gv601: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape165: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape164, gv601, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape164) gv602: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc424: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv602, R.dtype("float16")) _341: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape165, alloc424) R.vm.kill_object(reshape165) gv603: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape166: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc424, gv603, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc424) gv604: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape167: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape166, gv604, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape166) model_layers_5_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[32] gv605: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc425: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv605, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape167, model_layers_5_self_attn_o_proj_weight3, alloc425) R.vm.kill_object(reshape167) R.vm.kill_object(model_layers_5_self_attn_o_proj_weight3) model_layers_5_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[36] gv606: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc426: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv606, R.dtype("float16")) gv607: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc427: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv607, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc425, alloc422, model_layers_5_post_attention_layernorm_weight3, alloc426, alloc427) R.vm.kill_object(alloc422) R.vm.kill_object(alloc425) R.vm.kill_object(model_layers_5_post_attention_layernorm_weight3) lv92: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc426, alloc427 model_layers_5_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[33] gv608: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc428: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv608, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc426, model_layers_5_mlp_gate_up_proj_weight3, alloc428) R.vm.kill_object(alloc426) R.vm.kill_object(model_layers_5_mlp_gate_up_proj_weight3) gv609: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc429: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv609, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc428, alloc429) R.vm.kill_object(alloc428) model_layers_5_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[34] gv610: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc430: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv610, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc429, model_layers_5_mlp_down_proj_weight3, alloc430) R.vm.kill_object(alloc429) R.vm.kill_object(model_layers_5_mlp_down_proj_weight3) model_layers_6_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[41] gv611: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc431: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv611, R.dtype("float16")) gv612: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc432: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv612, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc430, alloc427, model_layers_6_input_layernorm_weight3, alloc431, alloc432) R.vm.kill_object(alloc427) R.vm.kill_object(alloc430) R.vm.kill_object(model_layers_6_input_layernorm_weight3) lv94: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc431, alloc432 model_layers_6_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[37] gv613: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc433: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv613, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc431, model_layers_6_self_attn_qkv_proj_weight3, alloc433) R.vm.kill_object(alloc431) R.vm.kill_object(model_layers_6_self_attn_qkv_proj_weight3) gv614: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape168: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc433, gv614, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc433) gv615: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape169: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape168, gv615, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape168) gv616: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc434: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv616, R.dtype("float16")) _349: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(6), R.prim_value(T.float32(1.0)), reshape169, alloc434) R.vm.kill_object(reshape169) gv617: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape170: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc434, gv617, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc434) gv618: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape171: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape170, gv618, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape170) model_layers_6_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[38] gv619: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc435: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv619, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape171, model_layers_6_self_attn_o_proj_weight3, alloc435) R.vm.kill_object(reshape171) R.vm.kill_object(model_layers_6_self_attn_o_proj_weight3) model_layers_6_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[42] gv620: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc436: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv620, R.dtype("float16")) gv621: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc437: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv621, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc435, alloc432, model_layers_6_post_attention_layernorm_weight3, alloc436, alloc437) R.vm.kill_object(alloc432) R.vm.kill_object(alloc435) R.vm.kill_object(model_layers_6_post_attention_layernorm_weight3) lv96: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc436, alloc437 model_layers_6_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[39] gv622: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc438: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv622, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc436, model_layers_6_mlp_gate_up_proj_weight3, alloc438) R.vm.kill_object(alloc436) R.vm.kill_object(model_layers_6_mlp_gate_up_proj_weight3) gv623: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc439: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv623, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc438, alloc439) R.vm.kill_object(alloc438) model_layers_6_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[40] gv624: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc440: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv624, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc439, model_layers_6_mlp_down_proj_weight3, alloc440) R.vm.kill_object(alloc439) R.vm.kill_object(model_layers_6_mlp_down_proj_weight3) model_layers_7_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[47] gv625: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc441: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv625, R.dtype("float16")) gv626: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc442: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv626, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc440, alloc437, model_layers_7_input_layernorm_weight3, alloc441, alloc442) R.vm.kill_object(alloc437) R.vm.kill_object(alloc440) R.vm.kill_object(model_layers_7_input_layernorm_weight3) lv98: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc441, alloc442 model_layers_7_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[43] gv627: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc443: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv627, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc441, model_layers_7_self_attn_qkv_proj_weight3, alloc443) R.vm.kill_object(alloc441) R.vm.kill_object(model_layers_7_self_attn_qkv_proj_weight3) gv628: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape172: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc443, gv628, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc443) gv629: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape173: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape172, gv629, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape172) gv630: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc444: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv630, R.dtype("float16")) _357: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(7), R.prim_value(T.float32(1.0)), reshape173, alloc444) R.vm.kill_object(reshape173) gv631: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape174: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc444, gv631, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc444) gv632: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape175: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape174, gv632, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape174) model_layers_7_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[44] gv633: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc445: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv633, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape175, model_layers_7_self_attn_o_proj_weight3, alloc445) R.vm.kill_object(reshape175) R.vm.kill_object(model_layers_7_self_attn_o_proj_weight3) model_layers_7_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[48] gv634: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc446: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv634, R.dtype("float16")) gv635: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc447: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv635, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc445, alloc442, model_layers_7_post_attention_layernorm_weight3, alloc446, alloc447) R.vm.kill_object(alloc442) R.vm.kill_object(alloc445) R.vm.kill_object(model_layers_7_post_attention_layernorm_weight3) lv100: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc446, alloc447 model_layers_7_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[45] gv636: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc448: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv636, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc446, model_layers_7_mlp_gate_up_proj_weight3, alloc448) R.vm.kill_object(alloc446) R.vm.kill_object(model_layers_7_mlp_gate_up_proj_weight3) gv637: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc449: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv637, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc448, alloc449) R.vm.kill_object(alloc448) model_layers_7_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[46] gv638: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc450: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv638, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc449, model_layers_7_mlp_down_proj_weight3, alloc450) R.vm.kill_object(alloc449) R.vm.kill_object(model_layers_7_mlp_down_proj_weight3) model_layers_8_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[53] gv639: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc451: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv639, R.dtype("float16")) gv640: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc452: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv640, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc450, alloc447, model_layers_8_input_layernorm_weight3, alloc451, alloc452) R.vm.kill_object(alloc447) R.vm.kill_object(alloc450) R.vm.kill_object(model_layers_8_input_layernorm_weight3) lv102: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc451, alloc452 model_layers_8_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[49] gv641: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc453: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv641, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc451, model_layers_8_self_attn_qkv_proj_weight3, alloc453) R.vm.kill_object(alloc451) R.vm.kill_object(model_layers_8_self_attn_qkv_proj_weight3) gv642: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape176: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc453, gv642, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc453) gv643: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape177: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape176, gv643, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape176) gv644: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc454: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv644, R.dtype("float16")) _365: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(8), R.prim_value(T.float32(1.0)), reshape177, alloc454) R.vm.kill_object(reshape177) gv645: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape178: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc454, gv645, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc454) gv646: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape179: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape178, gv646, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape178) model_layers_8_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[50] gv647: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc455: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv647, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape179, model_layers_8_self_attn_o_proj_weight3, alloc455) R.vm.kill_object(reshape179) R.vm.kill_object(model_layers_8_self_attn_o_proj_weight3) model_layers_8_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[54] gv648: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc456: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv648, R.dtype("float16")) gv649: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc457: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv649, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc455, alloc452, model_layers_8_post_attention_layernorm_weight3, alloc456, alloc457) R.vm.kill_object(alloc452) R.vm.kill_object(alloc455) R.vm.kill_object(model_layers_8_post_attention_layernorm_weight3) lv104: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc456, alloc457 model_layers_8_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[51] gv650: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc458: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv650, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc456, model_layers_8_mlp_gate_up_proj_weight3, alloc458) R.vm.kill_object(alloc456) R.vm.kill_object(model_layers_8_mlp_gate_up_proj_weight3) gv651: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc459: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv651, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc458, alloc459) R.vm.kill_object(alloc458) model_layers_8_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[52] gv652: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc460: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv652, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc459, model_layers_8_mlp_down_proj_weight3, alloc460) R.vm.kill_object(alloc459) R.vm.kill_object(model_layers_8_mlp_down_proj_weight3) model_layers_9_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[59] gv653: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc461: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv653, R.dtype("float16")) gv654: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc462: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv654, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc460, alloc457, model_layers_9_input_layernorm_weight3, alloc461, alloc462) R.vm.kill_object(alloc457) R.vm.kill_object(alloc460) R.vm.kill_object(model_layers_9_input_layernorm_weight3) lv106: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc461, alloc462 model_layers_9_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[55] gv655: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc463: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv655, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc461, model_layers_9_self_attn_qkv_proj_weight3, alloc463) R.vm.kill_object(alloc461) R.vm.kill_object(model_layers_9_self_attn_qkv_proj_weight3) gv656: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape180: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc463, gv656, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc463) gv657: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape181: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape180, gv657, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape180) gv658: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc464: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv658, R.dtype("float16")) _373: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(9), R.prim_value(T.float32(1.0)), reshape181, alloc464) R.vm.kill_object(reshape181) gv659: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape182: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc464, gv659, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc464) gv660: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape183: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape182, gv660, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape182) model_layers_9_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[56] gv661: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc465: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv661, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape183, model_layers_9_self_attn_o_proj_weight3, alloc465) R.vm.kill_object(reshape183) R.vm.kill_object(model_layers_9_self_attn_o_proj_weight3) model_layers_9_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[60] gv662: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc466: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv662, R.dtype("float16")) gv663: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc467: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv663, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc465, alloc462, model_layers_9_post_attention_layernorm_weight3, alloc466, alloc467) R.vm.kill_object(alloc462) R.vm.kill_object(alloc465) R.vm.kill_object(model_layers_9_post_attention_layernorm_weight3) lv108: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc466, alloc467 model_layers_9_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[57] gv664: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc468: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv664, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc466, model_layers_9_mlp_gate_up_proj_weight3, alloc468) R.vm.kill_object(alloc466) R.vm.kill_object(model_layers_9_mlp_gate_up_proj_weight3) gv665: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc469: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv665, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc468, alloc469) R.vm.kill_object(alloc468) model_layers_9_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[58] gv666: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc470: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv666, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc469, model_layers_9_mlp_down_proj_weight3, alloc470) R.vm.kill_object(alloc469) R.vm.kill_object(model_layers_9_mlp_down_proj_weight3) model_layers_10_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[65] gv667: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc471: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv667, R.dtype("float16")) gv668: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc472: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv668, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc470, alloc467, model_layers_10_input_layernorm_weight3, alloc471, alloc472) R.vm.kill_object(alloc467) R.vm.kill_object(alloc470) R.vm.kill_object(model_layers_10_input_layernorm_weight3) lv110: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc471, alloc472 model_layers_10_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[61] gv669: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc473: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv669, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc471, model_layers_10_self_attn_qkv_proj_weight3, alloc473) R.vm.kill_object(alloc471) R.vm.kill_object(model_layers_10_self_attn_qkv_proj_weight3) gv670: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape184: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc473, gv670, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc473) gv671: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape185: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape184, gv671, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape184) gv672: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc474: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv672, R.dtype("float16")) _381: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(10), R.prim_value(T.float32(1.0)), reshape185, alloc474) R.vm.kill_object(reshape185) gv673: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape186: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc474, gv673, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc474) gv674: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape187: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape186, gv674, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape186) model_layers_10_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[62] gv675: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc475: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv675, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape187, model_layers_10_self_attn_o_proj_weight3, alloc475) R.vm.kill_object(reshape187) R.vm.kill_object(model_layers_10_self_attn_o_proj_weight3) model_layers_10_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[66] gv676: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc476: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv676, R.dtype("float16")) gv677: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc477: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv677, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc475, alloc472, model_layers_10_post_attention_layernorm_weight3, alloc476, alloc477) R.vm.kill_object(alloc472) R.vm.kill_object(alloc475) R.vm.kill_object(model_layers_10_post_attention_layernorm_weight3) lv112: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc476, alloc477 model_layers_10_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[63] gv678: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc478: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv678, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc476, model_layers_10_mlp_gate_up_proj_weight3, alloc478) R.vm.kill_object(alloc476) R.vm.kill_object(model_layers_10_mlp_gate_up_proj_weight3) gv679: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc479: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv679, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc478, alloc479) R.vm.kill_object(alloc478) model_layers_10_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[64] gv680: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc480: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv680, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc479, model_layers_10_mlp_down_proj_weight3, alloc480) R.vm.kill_object(alloc479) R.vm.kill_object(model_layers_10_mlp_down_proj_weight3) model_layers_11_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[71] gv681: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc481: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv681, R.dtype("float16")) gv682: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc482: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv682, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc480, alloc477, model_layers_11_input_layernorm_weight3, alloc481, alloc482) R.vm.kill_object(alloc477) R.vm.kill_object(alloc480) R.vm.kill_object(model_layers_11_input_layernorm_weight3) lv114: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc481, alloc482 model_layers_11_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[67] gv683: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc483: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv683, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc481, model_layers_11_self_attn_qkv_proj_weight3, alloc483) R.vm.kill_object(alloc481) R.vm.kill_object(model_layers_11_self_attn_qkv_proj_weight3) gv684: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape188: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc483, gv684, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc483) gv685: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape189: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape188, gv685, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape188) gv686: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc484: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv686, R.dtype("float16")) _389: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(11), R.prim_value(T.float32(1.0)), reshape189, alloc484) R.vm.kill_object(reshape189) gv687: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape190: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc484, gv687, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc484) gv688: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape191: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape190, gv688, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape190) model_layers_11_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[68] gv689: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc485: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv689, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape191, model_layers_11_self_attn_o_proj_weight3, alloc485) R.vm.kill_object(reshape191) R.vm.kill_object(model_layers_11_self_attn_o_proj_weight3) model_layers_11_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[72] gv690: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc486: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv690, R.dtype("float16")) gv691: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc487: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv691, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc485, alloc482, model_layers_11_post_attention_layernorm_weight3, alloc486, alloc487) R.vm.kill_object(alloc482) R.vm.kill_object(alloc485) R.vm.kill_object(model_layers_11_post_attention_layernorm_weight3) lv116: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc486, alloc487 model_layers_11_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[69] gv692: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc488: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv692, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc486, model_layers_11_mlp_gate_up_proj_weight3, alloc488) R.vm.kill_object(alloc486) R.vm.kill_object(model_layers_11_mlp_gate_up_proj_weight3) gv693: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc489: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv693, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc488, alloc489) R.vm.kill_object(alloc488) model_layers_11_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[70] gv694: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc490: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv694, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc489, model_layers_11_mlp_down_proj_weight3, alloc490) R.vm.kill_object(alloc489) R.vm.kill_object(model_layers_11_mlp_down_proj_weight3) model_layers_12_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[77] gv695: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc491: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv695, R.dtype("float16")) gv696: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc492: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv696, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc490, alloc487, model_layers_12_input_layernorm_weight3, alloc491, alloc492) R.vm.kill_object(alloc487) R.vm.kill_object(alloc490) R.vm.kill_object(model_layers_12_input_layernorm_weight3) lv118: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc491, alloc492 model_layers_12_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[73] gv697: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc493: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv697, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc491, model_layers_12_self_attn_qkv_proj_weight3, alloc493) R.vm.kill_object(alloc491) R.vm.kill_object(model_layers_12_self_attn_qkv_proj_weight3) gv698: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape192: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc493, gv698, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc493) gv699: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape193: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape192, gv699, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape192) gv700: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc494: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv700, R.dtype("float16")) _397: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(12), R.prim_value(T.float32(1.0)), reshape193, alloc494) R.vm.kill_object(reshape193) gv701: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape194: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc494, gv701, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc494) gv702: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape195: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape194, gv702, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape194) model_layers_12_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[74] gv703: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc495: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv703, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape195, model_layers_12_self_attn_o_proj_weight3, alloc495) R.vm.kill_object(reshape195) R.vm.kill_object(model_layers_12_self_attn_o_proj_weight3) model_layers_12_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[78] gv704: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc496: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv704, R.dtype("float16")) gv705: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc497: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv705, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc495, alloc492, model_layers_12_post_attention_layernorm_weight3, alloc496, alloc497) R.vm.kill_object(alloc492) R.vm.kill_object(alloc495) R.vm.kill_object(model_layers_12_post_attention_layernorm_weight3) lv120: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc496, alloc497 model_layers_12_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[75] gv706: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc498: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv706, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc496, model_layers_12_mlp_gate_up_proj_weight3, alloc498) R.vm.kill_object(alloc496) R.vm.kill_object(model_layers_12_mlp_gate_up_proj_weight3) gv707: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc499: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv707, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc498, alloc499) R.vm.kill_object(alloc498) model_layers_12_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[76] gv708: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc500: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv708, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc499, model_layers_12_mlp_down_proj_weight3, alloc500) R.vm.kill_object(alloc499) R.vm.kill_object(model_layers_12_mlp_down_proj_weight3) model_layers_13_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[83] gv709: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc501: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv709, R.dtype("float16")) gv710: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc502: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv710, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc500, alloc497, model_layers_13_input_layernorm_weight3, alloc501, alloc502) R.vm.kill_object(alloc497) R.vm.kill_object(alloc500) R.vm.kill_object(model_layers_13_input_layernorm_weight3) lv122: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc501, alloc502 model_layers_13_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[79] gv711: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc503: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv711, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc501, model_layers_13_self_attn_qkv_proj_weight3, alloc503) R.vm.kill_object(alloc501) R.vm.kill_object(model_layers_13_self_attn_qkv_proj_weight3) gv712: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape196: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc503, gv712, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc503) gv713: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape197: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape196, gv713, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape196) gv714: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc504: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv714, R.dtype("float16")) _405: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(13), R.prim_value(T.float32(1.0)), reshape197, alloc504) R.vm.kill_object(reshape197) gv715: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape198: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc504, gv715, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc504) gv716: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape199: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape198, gv716, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape198) model_layers_13_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[80] gv717: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc505: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv717, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape199, model_layers_13_self_attn_o_proj_weight3, alloc505) R.vm.kill_object(reshape199) R.vm.kill_object(model_layers_13_self_attn_o_proj_weight3) model_layers_13_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[84] gv718: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc506: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv718, R.dtype("float16")) gv719: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc507: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv719, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc505, alloc502, model_layers_13_post_attention_layernorm_weight3, alloc506, alloc507) R.vm.kill_object(alloc502) R.vm.kill_object(alloc505) R.vm.kill_object(model_layers_13_post_attention_layernorm_weight3) lv124: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc506, alloc507 model_layers_13_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[81] gv720: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc508: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv720, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc506, model_layers_13_mlp_gate_up_proj_weight3, alloc508) R.vm.kill_object(alloc506) R.vm.kill_object(model_layers_13_mlp_gate_up_proj_weight3) gv721: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc509: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv721, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc508, alloc509) R.vm.kill_object(alloc508) model_layers_13_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[82] gv722: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc510: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv722, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc509, model_layers_13_mlp_down_proj_weight3, alloc510) R.vm.kill_object(alloc509) R.vm.kill_object(model_layers_13_mlp_down_proj_weight3) model_layers_14_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[89] gv723: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc511: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv723, R.dtype("float16")) gv724: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc512: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv724, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc510, alloc507, model_layers_14_input_layernorm_weight3, alloc511, alloc512) R.vm.kill_object(alloc507) R.vm.kill_object(alloc510) R.vm.kill_object(model_layers_14_input_layernorm_weight3) lv126: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc511, alloc512 model_layers_14_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[85] gv725: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc513: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv725, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc511, model_layers_14_self_attn_qkv_proj_weight3, alloc513) R.vm.kill_object(alloc511) R.vm.kill_object(model_layers_14_self_attn_qkv_proj_weight3) gv726: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape200: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc513, gv726, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc513) gv727: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape201: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape200, gv727, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape200) gv728: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc514: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv728, R.dtype("float16")) _413: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(14), R.prim_value(T.float32(1.0)), reshape201, alloc514) R.vm.kill_object(reshape201) gv729: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape202: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc514, gv729, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc514) gv730: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape203: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape202, gv730, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape202) model_layers_14_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[86] gv731: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc515: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv731, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape203, model_layers_14_self_attn_o_proj_weight3, alloc515) R.vm.kill_object(reshape203) R.vm.kill_object(model_layers_14_self_attn_o_proj_weight3) model_layers_14_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[90] gv732: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc516: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv732, R.dtype("float16")) gv733: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc517: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv733, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc515, alloc512, model_layers_14_post_attention_layernorm_weight3, alloc516, alloc517) R.vm.kill_object(alloc512) R.vm.kill_object(alloc515) R.vm.kill_object(model_layers_14_post_attention_layernorm_weight3) lv128: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc516, alloc517 model_layers_14_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[87] gv734: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc518: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv734, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc516, model_layers_14_mlp_gate_up_proj_weight3, alloc518) R.vm.kill_object(alloc516) R.vm.kill_object(model_layers_14_mlp_gate_up_proj_weight3) gv735: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc519: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv735, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc518, alloc519) R.vm.kill_object(alloc518) model_layers_14_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[88] gv736: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc520: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv736, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc519, model_layers_14_mlp_down_proj_weight3, alloc520) R.vm.kill_object(alloc519) R.vm.kill_object(model_layers_14_mlp_down_proj_weight3) model_layers_15_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[95] gv737: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc521: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv737, R.dtype("float16")) gv738: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc522: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv738, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc520, alloc517, model_layers_15_input_layernorm_weight3, alloc521, alloc522) R.vm.kill_object(alloc517) R.vm.kill_object(alloc520) R.vm.kill_object(model_layers_15_input_layernorm_weight3) lv130: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc521, alloc522 model_layers_15_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[91] gv739: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc523: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv739, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc521, model_layers_15_self_attn_qkv_proj_weight3, alloc523) R.vm.kill_object(alloc521) R.vm.kill_object(model_layers_15_self_attn_qkv_proj_weight3) gv740: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape204: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc523, gv740, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc523) gv741: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape205: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape204, gv741, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape204) gv742: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc524: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv742, R.dtype("float16")) _421: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(15), R.prim_value(T.float32(1.0)), reshape205, alloc524) R.vm.kill_object(reshape205) gv743: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape206: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc524, gv743, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc524) gv744: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape207: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape206, gv744, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape206) model_layers_15_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[92] gv745: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc525: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv745, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape207, model_layers_15_self_attn_o_proj_weight3, alloc525) R.vm.kill_object(reshape207) R.vm.kill_object(model_layers_15_self_attn_o_proj_weight3) model_layers_15_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[96] gv746: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc526: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv746, R.dtype("float16")) gv747: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc527: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv747, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc525, alloc522, model_layers_15_post_attention_layernorm_weight3, alloc526, alloc527) R.vm.kill_object(alloc522) R.vm.kill_object(alloc525) R.vm.kill_object(model_layers_15_post_attention_layernorm_weight3) lv132: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc526, alloc527 model_layers_15_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[93] gv748: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc528: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv748, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc526, model_layers_15_mlp_gate_up_proj_weight3, alloc528) R.vm.kill_object(alloc526) R.vm.kill_object(model_layers_15_mlp_gate_up_proj_weight3) gv749: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc529: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv749, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc528, alloc529) R.vm.kill_object(alloc528) model_layers_15_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[94] gv750: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc530: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv750, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc529, model_layers_15_mlp_down_proj_weight3, alloc530) R.vm.kill_object(alloc529) R.vm.kill_object(model_layers_15_mlp_down_proj_weight3) model_layers_16_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[101] gv751: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc531: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv751, R.dtype("float16")) gv752: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc532: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv752, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc530, alloc527, model_layers_16_input_layernorm_weight3, alloc531, alloc532) R.vm.kill_object(alloc527) R.vm.kill_object(alloc530) R.vm.kill_object(model_layers_16_input_layernorm_weight3) lv134: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc531, alloc532 model_layers_16_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[97] gv753: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc533: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv753, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc531, model_layers_16_self_attn_qkv_proj_weight3, alloc533) R.vm.kill_object(alloc531) R.vm.kill_object(model_layers_16_self_attn_qkv_proj_weight3) gv754: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape208: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc533, gv754, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc533) gv755: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape209: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape208, gv755, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape208) gv756: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc534: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv756, R.dtype("float16")) _429: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(16), R.prim_value(T.float32(1.0)), reshape209, alloc534) R.vm.kill_object(reshape209) gv757: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape210: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc534, gv757, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc534) gv758: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape211: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape210, gv758, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape210) model_layers_16_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[98] gv759: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc535: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv759, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape211, model_layers_16_self_attn_o_proj_weight3, alloc535) R.vm.kill_object(reshape211) R.vm.kill_object(model_layers_16_self_attn_o_proj_weight3) model_layers_16_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[102] gv760: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc536: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv760, R.dtype("float16")) gv761: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc537: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv761, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc535, alloc532, model_layers_16_post_attention_layernorm_weight3, alloc536, alloc537) R.vm.kill_object(alloc532) R.vm.kill_object(alloc535) R.vm.kill_object(model_layers_16_post_attention_layernorm_weight3) lv136: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc536, alloc537 model_layers_16_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[99] gv762: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc538: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv762, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc536, model_layers_16_mlp_gate_up_proj_weight3, alloc538) R.vm.kill_object(alloc536) R.vm.kill_object(model_layers_16_mlp_gate_up_proj_weight3) gv763: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc539: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv763, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc538, alloc539) R.vm.kill_object(alloc538) model_layers_16_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[100] gv764: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc540: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv764, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc539, model_layers_16_mlp_down_proj_weight3, alloc540) R.vm.kill_object(alloc539) R.vm.kill_object(model_layers_16_mlp_down_proj_weight3) model_layers_17_input_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[107] gv765: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc541: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv765, R.dtype("float16")) gv766: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc542: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv766, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc540, alloc537, model_layers_17_input_layernorm_weight3, alloc541, alloc542) R.vm.kill_object(alloc537) R.vm.kill_object(alloc540) R.vm.kill_object(model_layers_17_input_layernorm_weight3) lv138: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc541, alloc542 model_layers_17_self_attn_qkv_proj_weight3: R.Tensor((2560, 2048), dtype="float16") = packed_params[103] gv767: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc543: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv767, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc541, model_layers_17_self_attn_qkv_proj_weight3, alloc543) R.vm.kill_object(alloc541) R.vm.kill_object(model_layers_17_self_attn_qkv_proj_weight3) gv768: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape212: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc543, gv768, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc543) gv769: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape213: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape212, gv769, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape212) gv770: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc544: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv770, R.dtype("float16")) _437: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(17), R.prim_value(T.float32(1.0)), reshape213, alloc544) R.vm.kill_object(reshape213) gv771: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape214: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc544, gv771, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc544) gv772: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape215: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape214, gv772, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape214) model_layers_17_self_attn_o_proj_weight3: R.Tensor((2048, 2048), dtype="float16") = packed_params[104] gv773: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc545: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv773, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape215, model_layers_17_self_attn_o_proj_weight3, alloc545) R.vm.kill_object(reshape215) R.vm.kill_object(model_layers_17_self_attn_o_proj_weight3) model_layers_17_post_attention_layernorm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[108] gv774: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc546: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv774, R.dtype("float16")) gv775: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc547: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage16, R.prim_value(0), gv775, R.dtype("float16")) R.vm.kill_object(storage16) cls.gemma_q0f16_fuse_add_norm_prefill(alloc545, alloc542, model_layers_17_post_attention_layernorm_weight3, alloc546, alloc547) R.vm.kill_object(alloc542) R.vm.kill_object(alloc545) R.vm.kill_object(model_layers_17_post_attention_layernorm_weight3) lv140: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc546, alloc547 model_layers_17_mlp_gate_up_proj_weight3: R.Tensor((32768, 2048), dtype="float16") = packed_params[105] gv776: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc548: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv776, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc546, model_layers_17_mlp_gate_up_proj_weight3, alloc548) R.vm.kill_object(alloc546) R.vm.kill_object(model_layers_17_mlp_gate_up_proj_weight3) gv777: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc549: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv777, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc548, alloc549) R.vm.kill_object(alloc548) model_layers_17_mlp_down_proj_weight3: R.Tensor((2048, 16384), dtype="float16") = packed_params[106] gv778: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc550: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage15, R.prim_value(0), gv778, R.dtype("float16")) R.vm.kill_object(storage15) cls.gemma_q0f16_NT_matmul8(alloc549, model_layers_17_mlp_down_proj_weight3, alloc550) R.vm.kill_object(alloc549) R.vm.kill_object(model_layers_17_mlp_down_proj_weight3) model_norm_weight3: R.Tensor((2048,), dtype="float16") = packed_params[109] gv779: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc551: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage18, R.prim_value(0), gv779, R.dtype("float16")) R.vm.kill_object(storage18) gv780: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc552: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage17, R.prim_value(0), gv780, R.dtype("float16")) R.vm.kill_object(storage17) cls.gemma_q0f16_fuse_add_norm_prefill(alloc550, alloc547, model_norm_weight3, alloc551, alloc552) R.vm.kill_object(alloc547) R.vm.kill_object(alloc550) R.vm.kill_object(model_norm_weight3) lv142: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc551, alloc552 R.vm.kill_object(alloc552) storage19: R.Object = R.vm.alloc_storage(R.shape([524288]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv781: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc553: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage19, R.prim_value(0), gv781, R.dtype("float16")) R.vm.kill_object(storage19) cls.gemma_q0f16_take(alloc551, logit_positions, alloc553) R.vm.kill_object(alloc551) model_embed_tokens_weight3: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[0] gv782: R.Shape(ndim=1) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(4), sinfo_args=(R.Shape(ndim=1),)) storage20: R.Object = R.vm.alloc_storage(gv782, R.prim_value(0), R.dtype("uint8"), R.str("global")) gv783: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(1), R.prim_value(1), R.prim_value(2), sinfo_args=(R.Shape(ndim=3),)) alloc554: R.Tensor(dtype="float32", ndim=3) = R.vm.alloc_tensor(storage20, R.prim_value(0), gv783, R.dtype("float32")) R.vm.kill_object(storage20) cls.gemma_q0f16_NT_matmul9(alloc553, model_embed_tokens_weight3, alloc554) R.vm.kill_object(alloc553) R.vm.kill_object(model_embed_tokens_weight3) gv3: R.Tuple(R.Tensor(dtype="float32", ndim=3), R.Object) = alloc554, paged_kv_cache R.vm.kill_object(alloc554) gv784: R.Tensor(dtype="float32", ndim=3) = gv3[0] R.call_packed("vm.builtin.match_shape", gv784, shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(3), R.prim_value(1), R.prim_value(3), R.prim_value(2), R.str("ErrorContext(fn=batch_prefill, loc=return, annotation=R.Tuple(R.Tensor((1, batch_size, vocab_size), dtype=\"float32\"), R.Object)) "), sinfo_args=(R.Tuple,)) return gv3 @R.function def batch_verify(input_embeds: R.Tensor((1, "seq_len", 2048), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 2048), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"))) -> R.Tuple(R.Tensor((1, "seq_len", "vocab_size"), dtype="float32"), R.Object): seq_len = T.int64() vocab_size = T.int64() R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.force_pure": True, "relax.rewrite_cuda_graph.capture_symbolic_vars": ["batch_size", "seq_len"], "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 8192, "total_seq_len": 8192}}) cls = Module shape_heap: R.Tensor(dtype="int64", ndim=1) = R.call_builtin_with_ctx("vm.builtin.alloc_shape_heap", (R.prim_value(4),), sinfo_args=(R.Tensor(dtype="int64", ndim=1),)) R.call_packed("vm.builtin.check_tensor_info", input_embeds, R.prim_value(3), R.dtype("float16"), R.str("ErrorContext(fn=batch_verify, loc=param[0], param=input_embeds, annotation=R.Tensor((1, seq_len, 2048), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_tuple_info", packed_params, R.prim_value(110), R.str("ErrorContext(fn=batch_verify, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 2048), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv789: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[0] R.call_packed("vm.builtin.check_tensor_info", gv789, R.prim_value(2), R.dtype("float16"), R.str("ErrorContext(fn=batch_verify, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 2048), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", input_embeds, shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), R.str("ErrorContext(fn=batch_verify, loc=param[0], param=input_embeds, annotation=R.Tensor((1, seq_len, 2048), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv789, shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(2048), R.str("ErrorContext(fn=batch_verify, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 2048), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) cls.gemma_q0f16_shape_func5(shape_heap) storage26: R.Object = R.vm.alloc_storage(R.shape([33554432]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv790: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc738: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv790, R.dtype("float16")) cls.gemma_q0f16_multiply2(input_embeds, alloc738) model_layers_0_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[5] storage27: R.Object = R.vm.alloc_storage(R.shape([268435456]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv791: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc739: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv791, R.dtype("float16")) cls.gemma_q0f16_rms_norm1(alloc738, model_layers_0_input_layernorm_weight5, alloc739) R.vm.kill_object(model_layers_0_input_layernorm_weight5) model_layers_0_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[1] storage28: R.Object = R.vm.alloc_storage(R.shape([536870912]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv792: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc740: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv792, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc739, model_layers_0_self_attn_qkv_proj_weight5, alloc740) R.vm.kill_object(alloc739) R.vm.kill_object(model_layers_0_self_attn_qkv_proj_weight5) gv793: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape288: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc740, gv793, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc740) gv794: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape289: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape288, gv794, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape288) gv795: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc741: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv795, R.dtype("float16")) _596: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape289, alloc741) R.vm.kill_object(reshape289) gv796: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape290: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc741, gv796, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc741) gv797: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape291: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape290, gv797, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape290) model_layers_0_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[2] gv798: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc742: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv798, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape291, model_layers_0_self_attn_o_proj_weight5, alloc742) R.vm.kill_object(reshape291) R.vm.kill_object(model_layers_0_self_attn_o_proj_weight5) model_layers_0_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[6] gv799: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc743: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv799, R.dtype("float16")) storage29: R.Object = R.vm.alloc_storage(R.shape([268435456]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv800: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc744: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv800, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc742, alloc738, model_layers_0_post_attention_layernorm_weight5, alloc743, alloc744) R.vm.kill_object(alloc738) R.vm.kill_object(alloc742) R.vm.kill_object(model_layers_0_post_attention_layernorm_weight5) lv144: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc743, alloc744 model_layers_0_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[3] gv801: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc745: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv801, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc743, model_layers_0_mlp_gate_up_proj_weight5, alloc745) R.vm.kill_object(alloc743) R.vm.kill_object(model_layers_0_mlp_gate_up_proj_weight5) gv802: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc746: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv802, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc745, alloc746) R.vm.kill_object(alloc745) model_layers_0_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[4] gv803: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc747: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv803, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc746, model_layers_0_mlp_down_proj_weight5, alloc747) R.vm.kill_object(alloc746) R.vm.kill_object(model_layers_0_mlp_down_proj_weight5) model_layers_1_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[11] gv804: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc748: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv804, R.dtype("float16")) gv805: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc749: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv805, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc747, alloc744, model_layers_1_input_layernorm_weight5, alloc748, alloc749) R.vm.kill_object(alloc744) R.vm.kill_object(alloc747) R.vm.kill_object(model_layers_1_input_layernorm_weight5) lv146: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc748, alloc749 model_layers_1_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[7] gv806: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc750: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv806, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc748, model_layers_1_self_attn_qkv_proj_weight5, alloc750) R.vm.kill_object(alloc748) R.vm.kill_object(model_layers_1_self_attn_qkv_proj_weight5) gv807: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape292: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc750, gv807, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc750) gv808: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape293: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape292, gv808, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape292) gv809: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc751: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv809, R.dtype("float16")) _604: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape293, alloc751) R.vm.kill_object(reshape293) gv810: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape294: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc751, gv810, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc751) gv811: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape295: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape294, gv811, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape294) model_layers_1_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[8] gv812: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc752: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv812, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape295, model_layers_1_self_attn_o_proj_weight5, alloc752) R.vm.kill_object(reshape295) R.vm.kill_object(model_layers_1_self_attn_o_proj_weight5) model_layers_1_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[12] gv813: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc753: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv813, R.dtype("float16")) gv814: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc754: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv814, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc752, alloc749, model_layers_1_post_attention_layernorm_weight5, alloc753, alloc754) R.vm.kill_object(alloc749) R.vm.kill_object(alloc752) R.vm.kill_object(model_layers_1_post_attention_layernorm_weight5) lv148: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc753, alloc754 model_layers_1_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[9] gv815: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc755: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv815, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc753, model_layers_1_mlp_gate_up_proj_weight5, alloc755) R.vm.kill_object(alloc753) R.vm.kill_object(model_layers_1_mlp_gate_up_proj_weight5) gv816: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc756: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv816, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc755, alloc756) R.vm.kill_object(alloc755) model_layers_1_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[10] gv817: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc757: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv817, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc756, model_layers_1_mlp_down_proj_weight5, alloc757) R.vm.kill_object(alloc756) R.vm.kill_object(model_layers_1_mlp_down_proj_weight5) model_layers_2_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[17] gv818: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc758: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv818, R.dtype("float16")) gv819: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc759: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv819, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc757, alloc754, model_layers_2_input_layernorm_weight5, alloc758, alloc759) R.vm.kill_object(alloc754) R.vm.kill_object(alloc757) R.vm.kill_object(model_layers_2_input_layernorm_weight5) lv150: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc758, alloc759 model_layers_2_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[13] gv820: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc760: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv820, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc758, model_layers_2_self_attn_qkv_proj_weight5, alloc760) R.vm.kill_object(alloc758) R.vm.kill_object(model_layers_2_self_attn_qkv_proj_weight5) gv821: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape296: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc760, gv821, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc760) gv822: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape297: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape296, gv822, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape296) gv823: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc761: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv823, R.dtype("float16")) _612: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape297, alloc761) R.vm.kill_object(reshape297) gv824: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape298: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc761, gv824, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc761) gv825: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape299: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape298, gv825, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape298) model_layers_2_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[14] gv826: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc762: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv826, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape299, model_layers_2_self_attn_o_proj_weight5, alloc762) R.vm.kill_object(reshape299) R.vm.kill_object(model_layers_2_self_attn_o_proj_weight5) model_layers_2_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[18] gv827: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc763: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv827, R.dtype("float16")) gv828: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc764: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv828, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc762, alloc759, model_layers_2_post_attention_layernorm_weight5, alloc763, alloc764) R.vm.kill_object(alloc759) R.vm.kill_object(alloc762) R.vm.kill_object(model_layers_2_post_attention_layernorm_weight5) lv152: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc763, alloc764 model_layers_2_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[15] gv829: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc765: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv829, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc763, model_layers_2_mlp_gate_up_proj_weight5, alloc765) R.vm.kill_object(alloc763) R.vm.kill_object(model_layers_2_mlp_gate_up_proj_weight5) gv830: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc766: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv830, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc765, alloc766) R.vm.kill_object(alloc765) model_layers_2_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[16] gv831: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc767: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv831, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc766, model_layers_2_mlp_down_proj_weight5, alloc767) R.vm.kill_object(alloc766) R.vm.kill_object(model_layers_2_mlp_down_proj_weight5) model_layers_3_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[23] gv832: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc768: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv832, R.dtype("float16")) gv833: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc769: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv833, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc767, alloc764, model_layers_3_input_layernorm_weight5, alloc768, alloc769) R.vm.kill_object(alloc764) R.vm.kill_object(alloc767) R.vm.kill_object(model_layers_3_input_layernorm_weight5) lv154: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc768, alloc769 model_layers_3_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[19] gv834: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc770: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv834, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc768, model_layers_3_self_attn_qkv_proj_weight5, alloc770) R.vm.kill_object(alloc768) R.vm.kill_object(model_layers_3_self_attn_qkv_proj_weight5) gv835: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape300: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc770, gv835, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc770) gv836: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape301: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape300, gv836, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape300) gv837: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc771: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv837, R.dtype("float16")) _620: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape301, alloc771) R.vm.kill_object(reshape301) gv838: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape302: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc771, gv838, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc771) gv839: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape303: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape302, gv839, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape302) model_layers_3_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[20] gv840: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc772: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv840, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape303, model_layers_3_self_attn_o_proj_weight5, alloc772) R.vm.kill_object(reshape303) R.vm.kill_object(model_layers_3_self_attn_o_proj_weight5) model_layers_3_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[24] gv841: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc773: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv841, R.dtype("float16")) gv842: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc774: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv842, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc772, alloc769, model_layers_3_post_attention_layernorm_weight5, alloc773, alloc774) R.vm.kill_object(alloc769) R.vm.kill_object(alloc772) R.vm.kill_object(model_layers_3_post_attention_layernorm_weight5) lv156: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc773, alloc774 model_layers_3_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[21] gv843: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc775: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv843, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc773, model_layers_3_mlp_gate_up_proj_weight5, alloc775) R.vm.kill_object(alloc773) R.vm.kill_object(model_layers_3_mlp_gate_up_proj_weight5) gv844: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc776: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv844, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc775, alloc776) R.vm.kill_object(alloc775) model_layers_3_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[22] gv845: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc777: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv845, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc776, model_layers_3_mlp_down_proj_weight5, alloc777) R.vm.kill_object(alloc776) R.vm.kill_object(model_layers_3_mlp_down_proj_weight5) model_layers_4_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[29] gv846: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc778: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv846, R.dtype("float16")) gv847: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc779: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv847, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc777, alloc774, model_layers_4_input_layernorm_weight5, alloc778, alloc779) R.vm.kill_object(alloc774) R.vm.kill_object(alloc777) R.vm.kill_object(model_layers_4_input_layernorm_weight5) lv158: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc778, alloc779 model_layers_4_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[25] gv848: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc780: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv848, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc778, model_layers_4_self_attn_qkv_proj_weight5, alloc780) R.vm.kill_object(alloc778) R.vm.kill_object(model_layers_4_self_attn_qkv_proj_weight5) gv849: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape304: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc780, gv849, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc780) gv850: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape305: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape304, gv850, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape304) gv851: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc781: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv851, R.dtype("float16")) _628: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape305, alloc781) R.vm.kill_object(reshape305) gv852: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape306: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc781, gv852, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc781) gv853: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape307: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape306, gv853, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape306) model_layers_4_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[26] gv854: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc782: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv854, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape307, model_layers_4_self_attn_o_proj_weight5, alloc782) R.vm.kill_object(reshape307) R.vm.kill_object(model_layers_4_self_attn_o_proj_weight5) model_layers_4_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[30] gv855: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc783: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv855, R.dtype("float16")) gv856: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc784: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv856, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc782, alloc779, model_layers_4_post_attention_layernorm_weight5, alloc783, alloc784) R.vm.kill_object(alloc779) R.vm.kill_object(alloc782) R.vm.kill_object(model_layers_4_post_attention_layernorm_weight5) lv160: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc783, alloc784 model_layers_4_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[27] gv857: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc785: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv857, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc783, model_layers_4_mlp_gate_up_proj_weight5, alloc785) R.vm.kill_object(alloc783) R.vm.kill_object(model_layers_4_mlp_gate_up_proj_weight5) gv858: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc786: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv858, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc785, alloc786) R.vm.kill_object(alloc785) model_layers_4_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[28] gv859: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc787: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv859, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc786, model_layers_4_mlp_down_proj_weight5, alloc787) R.vm.kill_object(alloc786) R.vm.kill_object(model_layers_4_mlp_down_proj_weight5) model_layers_5_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[35] gv860: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc788: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv860, R.dtype("float16")) gv861: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc789: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv861, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc787, alloc784, model_layers_5_input_layernorm_weight5, alloc788, alloc789) R.vm.kill_object(alloc784) R.vm.kill_object(alloc787) R.vm.kill_object(model_layers_5_input_layernorm_weight5) lv162: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc788, alloc789 model_layers_5_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[31] gv862: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc790: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv862, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc788, model_layers_5_self_attn_qkv_proj_weight5, alloc790) R.vm.kill_object(alloc788) R.vm.kill_object(model_layers_5_self_attn_qkv_proj_weight5) gv863: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape308: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc790, gv863, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc790) gv864: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape309: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape308, gv864, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape308) gv865: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc791: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv865, R.dtype("float16")) _636: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape309, alloc791) R.vm.kill_object(reshape309) gv866: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape310: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc791, gv866, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc791) gv867: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape311: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape310, gv867, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape310) model_layers_5_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[32] gv868: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc792: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv868, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape311, model_layers_5_self_attn_o_proj_weight5, alloc792) R.vm.kill_object(reshape311) R.vm.kill_object(model_layers_5_self_attn_o_proj_weight5) model_layers_5_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[36] gv869: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc793: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv869, R.dtype("float16")) gv870: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc794: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv870, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc792, alloc789, model_layers_5_post_attention_layernorm_weight5, alloc793, alloc794) R.vm.kill_object(alloc789) R.vm.kill_object(alloc792) R.vm.kill_object(model_layers_5_post_attention_layernorm_weight5) lv164: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc793, alloc794 model_layers_5_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[33] gv871: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc795: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv871, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc793, model_layers_5_mlp_gate_up_proj_weight5, alloc795) R.vm.kill_object(alloc793) R.vm.kill_object(model_layers_5_mlp_gate_up_proj_weight5) gv872: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc796: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv872, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc795, alloc796) R.vm.kill_object(alloc795) model_layers_5_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[34] gv873: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc797: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv873, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc796, model_layers_5_mlp_down_proj_weight5, alloc797) R.vm.kill_object(alloc796) R.vm.kill_object(model_layers_5_mlp_down_proj_weight5) model_layers_6_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[41] gv874: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc798: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv874, R.dtype("float16")) gv875: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc799: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv875, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc797, alloc794, model_layers_6_input_layernorm_weight5, alloc798, alloc799) R.vm.kill_object(alloc794) R.vm.kill_object(alloc797) R.vm.kill_object(model_layers_6_input_layernorm_weight5) lv166: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc798, alloc799 model_layers_6_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[37] gv876: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc800: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv876, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc798, model_layers_6_self_attn_qkv_proj_weight5, alloc800) R.vm.kill_object(alloc798) R.vm.kill_object(model_layers_6_self_attn_qkv_proj_weight5) gv877: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape312: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc800, gv877, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc800) gv878: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape313: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape312, gv878, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape312) gv879: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc801: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv879, R.dtype("float16")) _644: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(6), R.prim_value(T.float32(1.0)), reshape313, alloc801) R.vm.kill_object(reshape313) gv880: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape314: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc801, gv880, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc801) gv881: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape315: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape314, gv881, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape314) model_layers_6_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[38] gv882: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc802: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv882, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape315, model_layers_6_self_attn_o_proj_weight5, alloc802) R.vm.kill_object(reshape315) R.vm.kill_object(model_layers_6_self_attn_o_proj_weight5) model_layers_6_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[42] gv883: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc803: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv883, R.dtype("float16")) gv884: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc804: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv884, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc802, alloc799, model_layers_6_post_attention_layernorm_weight5, alloc803, alloc804) R.vm.kill_object(alloc799) R.vm.kill_object(alloc802) R.vm.kill_object(model_layers_6_post_attention_layernorm_weight5) lv168: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc803, alloc804 model_layers_6_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[39] gv885: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc805: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv885, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc803, model_layers_6_mlp_gate_up_proj_weight5, alloc805) R.vm.kill_object(alloc803) R.vm.kill_object(model_layers_6_mlp_gate_up_proj_weight5) gv886: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc806: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv886, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc805, alloc806) R.vm.kill_object(alloc805) model_layers_6_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[40] gv887: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc807: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv887, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc806, model_layers_6_mlp_down_proj_weight5, alloc807) R.vm.kill_object(alloc806) R.vm.kill_object(model_layers_6_mlp_down_proj_weight5) model_layers_7_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[47] gv888: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc808: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv888, R.dtype("float16")) gv889: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc809: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv889, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc807, alloc804, model_layers_7_input_layernorm_weight5, alloc808, alloc809) R.vm.kill_object(alloc804) R.vm.kill_object(alloc807) R.vm.kill_object(model_layers_7_input_layernorm_weight5) lv170: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc808, alloc809 model_layers_7_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[43] gv890: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc810: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv890, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc808, model_layers_7_self_attn_qkv_proj_weight5, alloc810) R.vm.kill_object(alloc808) R.vm.kill_object(model_layers_7_self_attn_qkv_proj_weight5) gv891: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape316: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc810, gv891, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc810) gv892: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape317: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape316, gv892, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape316) gv893: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc811: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv893, R.dtype("float16")) _652: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(7), R.prim_value(T.float32(1.0)), reshape317, alloc811) R.vm.kill_object(reshape317) gv894: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape318: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc811, gv894, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc811) gv895: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape319: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape318, gv895, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape318) model_layers_7_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[44] gv896: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc812: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv896, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape319, model_layers_7_self_attn_o_proj_weight5, alloc812) R.vm.kill_object(reshape319) R.vm.kill_object(model_layers_7_self_attn_o_proj_weight5) model_layers_7_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[48] gv897: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc813: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv897, R.dtype("float16")) gv898: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc814: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv898, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc812, alloc809, model_layers_7_post_attention_layernorm_weight5, alloc813, alloc814) R.vm.kill_object(alloc809) R.vm.kill_object(alloc812) R.vm.kill_object(model_layers_7_post_attention_layernorm_weight5) lv172: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc813, alloc814 model_layers_7_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[45] gv899: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc815: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv899, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc813, model_layers_7_mlp_gate_up_proj_weight5, alloc815) R.vm.kill_object(alloc813) R.vm.kill_object(model_layers_7_mlp_gate_up_proj_weight5) gv900: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc816: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv900, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc815, alloc816) R.vm.kill_object(alloc815) model_layers_7_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[46] gv901: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc817: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv901, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc816, model_layers_7_mlp_down_proj_weight5, alloc817) R.vm.kill_object(alloc816) R.vm.kill_object(model_layers_7_mlp_down_proj_weight5) model_layers_8_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[53] gv902: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc818: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv902, R.dtype("float16")) gv903: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc819: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv903, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc817, alloc814, model_layers_8_input_layernorm_weight5, alloc818, alloc819) R.vm.kill_object(alloc814) R.vm.kill_object(alloc817) R.vm.kill_object(model_layers_8_input_layernorm_weight5) lv174: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc818, alloc819 model_layers_8_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[49] gv904: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc820: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv904, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc818, model_layers_8_self_attn_qkv_proj_weight5, alloc820) R.vm.kill_object(alloc818) R.vm.kill_object(model_layers_8_self_attn_qkv_proj_weight5) gv905: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape320: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc820, gv905, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc820) gv906: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape321: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape320, gv906, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape320) gv907: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc821: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv907, R.dtype("float16")) _660: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(8), R.prim_value(T.float32(1.0)), reshape321, alloc821) R.vm.kill_object(reshape321) gv908: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape322: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc821, gv908, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc821) gv909: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape323: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape322, gv909, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape322) model_layers_8_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[50] gv910: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc822: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv910, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape323, model_layers_8_self_attn_o_proj_weight5, alloc822) R.vm.kill_object(reshape323) R.vm.kill_object(model_layers_8_self_attn_o_proj_weight5) model_layers_8_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[54] gv911: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc823: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv911, R.dtype("float16")) gv912: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc824: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv912, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc822, alloc819, model_layers_8_post_attention_layernorm_weight5, alloc823, alloc824) R.vm.kill_object(alloc819) R.vm.kill_object(alloc822) R.vm.kill_object(model_layers_8_post_attention_layernorm_weight5) lv176: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc823, alloc824 model_layers_8_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[51] gv913: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc825: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv913, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc823, model_layers_8_mlp_gate_up_proj_weight5, alloc825) R.vm.kill_object(alloc823) R.vm.kill_object(model_layers_8_mlp_gate_up_proj_weight5) gv914: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc826: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv914, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc825, alloc826) R.vm.kill_object(alloc825) model_layers_8_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[52] gv915: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc827: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv915, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc826, model_layers_8_mlp_down_proj_weight5, alloc827) R.vm.kill_object(alloc826) R.vm.kill_object(model_layers_8_mlp_down_proj_weight5) model_layers_9_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[59] gv916: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc828: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv916, R.dtype("float16")) gv917: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc829: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv917, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc827, alloc824, model_layers_9_input_layernorm_weight5, alloc828, alloc829) R.vm.kill_object(alloc824) R.vm.kill_object(alloc827) R.vm.kill_object(model_layers_9_input_layernorm_weight5) lv178: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc828, alloc829 model_layers_9_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[55] gv918: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc830: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv918, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc828, model_layers_9_self_attn_qkv_proj_weight5, alloc830) R.vm.kill_object(alloc828) R.vm.kill_object(model_layers_9_self_attn_qkv_proj_weight5) gv919: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape324: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc830, gv919, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc830) gv920: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape325: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape324, gv920, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape324) gv921: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc831: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv921, R.dtype("float16")) _668: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(9), R.prim_value(T.float32(1.0)), reshape325, alloc831) R.vm.kill_object(reshape325) gv922: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape326: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc831, gv922, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc831) gv923: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape327: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape326, gv923, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape326) model_layers_9_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[56] gv924: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc832: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv924, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape327, model_layers_9_self_attn_o_proj_weight5, alloc832) R.vm.kill_object(reshape327) R.vm.kill_object(model_layers_9_self_attn_o_proj_weight5) model_layers_9_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[60] gv925: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc833: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv925, R.dtype("float16")) gv926: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc834: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv926, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc832, alloc829, model_layers_9_post_attention_layernorm_weight5, alloc833, alloc834) R.vm.kill_object(alloc829) R.vm.kill_object(alloc832) R.vm.kill_object(model_layers_9_post_attention_layernorm_weight5) lv180: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc833, alloc834 model_layers_9_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[57] gv927: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc835: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv927, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc833, model_layers_9_mlp_gate_up_proj_weight5, alloc835) R.vm.kill_object(alloc833) R.vm.kill_object(model_layers_9_mlp_gate_up_proj_weight5) gv928: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc836: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv928, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc835, alloc836) R.vm.kill_object(alloc835) model_layers_9_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[58] gv929: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc837: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv929, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc836, model_layers_9_mlp_down_proj_weight5, alloc837) R.vm.kill_object(alloc836) R.vm.kill_object(model_layers_9_mlp_down_proj_weight5) model_layers_10_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[65] gv930: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc838: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv930, R.dtype("float16")) gv931: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc839: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv931, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc837, alloc834, model_layers_10_input_layernorm_weight5, alloc838, alloc839) R.vm.kill_object(alloc834) R.vm.kill_object(alloc837) R.vm.kill_object(model_layers_10_input_layernorm_weight5) lv182: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc838, alloc839 model_layers_10_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[61] gv932: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc840: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv932, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc838, model_layers_10_self_attn_qkv_proj_weight5, alloc840) R.vm.kill_object(alloc838) R.vm.kill_object(model_layers_10_self_attn_qkv_proj_weight5) gv933: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape328: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc840, gv933, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc840) gv934: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape329: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape328, gv934, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape328) gv935: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc841: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv935, R.dtype("float16")) _676: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(10), R.prim_value(T.float32(1.0)), reshape329, alloc841) R.vm.kill_object(reshape329) gv936: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape330: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc841, gv936, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc841) gv937: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape331: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape330, gv937, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape330) model_layers_10_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[62] gv938: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc842: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv938, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape331, model_layers_10_self_attn_o_proj_weight5, alloc842) R.vm.kill_object(reshape331) R.vm.kill_object(model_layers_10_self_attn_o_proj_weight5) model_layers_10_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[66] gv939: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc843: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv939, R.dtype("float16")) gv940: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc844: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv940, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc842, alloc839, model_layers_10_post_attention_layernorm_weight5, alloc843, alloc844) R.vm.kill_object(alloc839) R.vm.kill_object(alloc842) R.vm.kill_object(model_layers_10_post_attention_layernorm_weight5) lv184: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc843, alloc844 model_layers_10_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[63] gv941: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc845: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv941, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc843, model_layers_10_mlp_gate_up_proj_weight5, alloc845) R.vm.kill_object(alloc843) R.vm.kill_object(model_layers_10_mlp_gate_up_proj_weight5) gv942: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc846: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv942, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc845, alloc846) R.vm.kill_object(alloc845) model_layers_10_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[64] gv943: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc847: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv943, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc846, model_layers_10_mlp_down_proj_weight5, alloc847) R.vm.kill_object(alloc846) R.vm.kill_object(model_layers_10_mlp_down_proj_weight5) model_layers_11_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[71] gv944: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc848: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv944, R.dtype("float16")) gv945: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc849: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv945, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc847, alloc844, model_layers_11_input_layernorm_weight5, alloc848, alloc849) R.vm.kill_object(alloc844) R.vm.kill_object(alloc847) R.vm.kill_object(model_layers_11_input_layernorm_weight5) lv186: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc848, alloc849 model_layers_11_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[67] gv946: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc850: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv946, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc848, model_layers_11_self_attn_qkv_proj_weight5, alloc850) R.vm.kill_object(alloc848) R.vm.kill_object(model_layers_11_self_attn_qkv_proj_weight5) gv947: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape332: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc850, gv947, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc850) gv948: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape333: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape332, gv948, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape332) gv949: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc851: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv949, R.dtype("float16")) _684: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(11), R.prim_value(T.float32(1.0)), reshape333, alloc851) R.vm.kill_object(reshape333) gv950: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape334: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc851, gv950, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc851) gv951: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape335: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape334, gv951, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape334) model_layers_11_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[68] gv952: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc852: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv952, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape335, model_layers_11_self_attn_o_proj_weight5, alloc852) R.vm.kill_object(reshape335) R.vm.kill_object(model_layers_11_self_attn_o_proj_weight5) model_layers_11_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[72] gv953: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc853: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv953, R.dtype("float16")) gv954: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc854: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv954, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc852, alloc849, model_layers_11_post_attention_layernorm_weight5, alloc853, alloc854) R.vm.kill_object(alloc849) R.vm.kill_object(alloc852) R.vm.kill_object(model_layers_11_post_attention_layernorm_weight5) lv188: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc853, alloc854 model_layers_11_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[69] gv955: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc855: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv955, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc853, model_layers_11_mlp_gate_up_proj_weight5, alloc855) R.vm.kill_object(alloc853) R.vm.kill_object(model_layers_11_mlp_gate_up_proj_weight5) gv956: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc856: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv956, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc855, alloc856) R.vm.kill_object(alloc855) model_layers_11_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[70] gv957: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc857: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv957, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc856, model_layers_11_mlp_down_proj_weight5, alloc857) R.vm.kill_object(alloc856) R.vm.kill_object(model_layers_11_mlp_down_proj_weight5) model_layers_12_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[77] gv958: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc858: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv958, R.dtype("float16")) gv959: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc859: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv959, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc857, alloc854, model_layers_12_input_layernorm_weight5, alloc858, alloc859) R.vm.kill_object(alloc854) R.vm.kill_object(alloc857) R.vm.kill_object(model_layers_12_input_layernorm_weight5) lv190: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc858, alloc859 model_layers_12_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[73] gv960: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc860: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv960, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc858, model_layers_12_self_attn_qkv_proj_weight5, alloc860) R.vm.kill_object(alloc858) R.vm.kill_object(model_layers_12_self_attn_qkv_proj_weight5) gv961: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape336: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc860, gv961, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc860) gv962: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape337: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape336, gv962, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape336) gv963: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc861: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv963, R.dtype("float16")) _692: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(12), R.prim_value(T.float32(1.0)), reshape337, alloc861) R.vm.kill_object(reshape337) gv964: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape338: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc861, gv964, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc861) gv965: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape339: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape338, gv965, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape338) model_layers_12_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[74] gv966: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc862: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv966, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape339, model_layers_12_self_attn_o_proj_weight5, alloc862) R.vm.kill_object(reshape339) R.vm.kill_object(model_layers_12_self_attn_o_proj_weight5) model_layers_12_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[78] gv967: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc863: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv967, R.dtype("float16")) gv968: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc864: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv968, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc862, alloc859, model_layers_12_post_attention_layernorm_weight5, alloc863, alloc864) R.vm.kill_object(alloc859) R.vm.kill_object(alloc862) R.vm.kill_object(model_layers_12_post_attention_layernorm_weight5) lv192: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc863, alloc864 model_layers_12_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[75] gv969: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc865: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv969, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc863, model_layers_12_mlp_gate_up_proj_weight5, alloc865) R.vm.kill_object(alloc863) R.vm.kill_object(model_layers_12_mlp_gate_up_proj_weight5) gv970: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc866: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv970, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc865, alloc866) R.vm.kill_object(alloc865) model_layers_12_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[76] gv971: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc867: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv971, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc866, model_layers_12_mlp_down_proj_weight5, alloc867) R.vm.kill_object(alloc866) R.vm.kill_object(model_layers_12_mlp_down_proj_weight5) model_layers_13_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[83] gv972: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc868: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv972, R.dtype("float16")) gv973: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc869: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv973, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc867, alloc864, model_layers_13_input_layernorm_weight5, alloc868, alloc869) R.vm.kill_object(alloc864) R.vm.kill_object(alloc867) R.vm.kill_object(model_layers_13_input_layernorm_weight5) lv194: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc868, alloc869 model_layers_13_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[79] gv974: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc870: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv974, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc868, model_layers_13_self_attn_qkv_proj_weight5, alloc870) R.vm.kill_object(alloc868) R.vm.kill_object(model_layers_13_self_attn_qkv_proj_weight5) gv975: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape340: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc870, gv975, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc870) gv976: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape341: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape340, gv976, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape340) gv977: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc871: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv977, R.dtype("float16")) _700: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(13), R.prim_value(T.float32(1.0)), reshape341, alloc871) R.vm.kill_object(reshape341) gv978: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape342: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc871, gv978, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc871) gv979: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape343: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape342, gv979, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape342) model_layers_13_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[80] gv980: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc872: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv980, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape343, model_layers_13_self_attn_o_proj_weight5, alloc872) R.vm.kill_object(reshape343) R.vm.kill_object(model_layers_13_self_attn_o_proj_weight5) model_layers_13_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[84] gv981: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc873: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv981, R.dtype("float16")) gv982: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc874: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv982, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc872, alloc869, model_layers_13_post_attention_layernorm_weight5, alloc873, alloc874) R.vm.kill_object(alloc869) R.vm.kill_object(alloc872) R.vm.kill_object(model_layers_13_post_attention_layernorm_weight5) lv196: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc873, alloc874 model_layers_13_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[81] gv983: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc875: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv983, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc873, model_layers_13_mlp_gate_up_proj_weight5, alloc875) R.vm.kill_object(alloc873) R.vm.kill_object(model_layers_13_mlp_gate_up_proj_weight5) gv984: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc876: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv984, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc875, alloc876) R.vm.kill_object(alloc875) model_layers_13_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[82] gv985: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc877: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv985, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc876, model_layers_13_mlp_down_proj_weight5, alloc877) R.vm.kill_object(alloc876) R.vm.kill_object(model_layers_13_mlp_down_proj_weight5) model_layers_14_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[89] gv986: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc878: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv986, R.dtype("float16")) gv987: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc879: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv987, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc877, alloc874, model_layers_14_input_layernorm_weight5, alloc878, alloc879) R.vm.kill_object(alloc874) R.vm.kill_object(alloc877) R.vm.kill_object(model_layers_14_input_layernorm_weight5) lv198: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc878, alloc879 model_layers_14_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[85] gv988: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc880: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv988, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc878, model_layers_14_self_attn_qkv_proj_weight5, alloc880) R.vm.kill_object(alloc878) R.vm.kill_object(model_layers_14_self_attn_qkv_proj_weight5) gv989: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape344: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc880, gv989, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc880) gv990: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape345: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape344, gv990, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape344) gv991: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc881: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv991, R.dtype("float16")) _708: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(14), R.prim_value(T.float32(1.0)), reshape345, alloc881) R.vm.kill_object(reshape345) gv992: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape346: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc881, gv992, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc881) gv993: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape347: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape346, gv993, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape346) model_layers_14_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[86] gv994: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc882: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv994, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape347, model_layers_14_self_attn_o_proj_weight5, alloc882) R.vm.kill_object(reshape347) R.vm.kill_object(model_layers_14_self_attn_o_proj_weight5) model_layers_14_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[90] gv995: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc883: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv995, R.dtype("float16")) gv996: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc884: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv996, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc882, alloc879, model_layers_14_post_attention_layernorm_weight5, alloc883, alloc884) R.vm.kill_object(alloc879) R.vm.kill_object(alloc882) R.vm.kill_object(model_layers_14_post_attention_layernorm_weight5) lv200: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc883, alloc884 model_layers_14_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[87] gv997: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc885: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv997, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc883, model_layers_14_mlp_gate_up_proj_weight5, alloc885) R.vm.kill_object(alloc883) R.vm.kill_object(model_layers_14_mlp_gate_up_proj_weight5) gv998: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc886: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv998, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc885, alloc886) R.vm.kill_object(alloc885) model_layers_14_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[88] gv999: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc887: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv999, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc886, model_layers_14_mlp_down_proj_weight5, alloc887) R.vm.kill_object(alloc886) R.vm.kill_object(model_layers_14_mlp_down_proj_weight5) model_layers_15_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[95] gv1000: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc888: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv1000, R.dtype("float16")) gv1001: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc889: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv1001, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc887, alloc884, model_layers_15_input_layernorm_weight5, alloc888, alloc889) R.vm.kill_object(alloc884) R.vm.kill_object(alloc887) R.vm.kill_object(model_layers_15_input_layernorm_weight5) lv202: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc888, alloc889 model_layers_15_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[91] gv1002: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc890: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv1002, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc888, model_layers_15_self_attn_qkv_proj_weight5, alloc890) R.vm.kill_object(alloc888) R.vm.kill_object(model_layers_15_self_attn_qkv_proj_weight5) gv1003: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape348: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc890, gv1003, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc890) gv1004: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape349: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape348, gv1004, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape348) gv1005: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc891: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1005, R.dtype("float16")) _716: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(15), R.prim_value(T.float32(1.0)), reshape349, alloc891) R.vm.kill_object(reshape349) gv1006: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape350: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc891, gv1006, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc891) gv1007: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape351: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape350, gv1007, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape350) model_layers_15_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[92] gv1008: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc892: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv1008, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape351, model_layers_15_self_attn_o_proj_weight5, alloc892) R.vm.kill_object(reshape351) R.vm.kill_object(model_layers_15_self_attn_o_proj_weight5) model_layers_15_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[96] gv1009: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc893: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1009, R.dtype("float16")) gv1010: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc894: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv1010, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc892, alloc889, model_layers_15_post_attention_layernorm_weight5, alloc893, alloc894) R.vm.kill_object(alloc889) R.vm.kill_object(alloc892) R.vm.kill_object(model_layers_15_post_attention_layernorm_weight5) lv204: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc893, alloc894 model_layers_15_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[93] gv1011: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc895: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv1011, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc893, model_layers_15_mlp_gate_up_proj_weight5, alloc895) R.vm.kill_object(alloc893) R.vm.kill_object(model_layers_15_mlp_gate_up_proj_weight5) gv1012: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc896: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv1012, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc895, alloc896) R.vm.kill_object(alloc895) model_layers_15_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[94] gv1013: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc897: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1013, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc896, model_layers_15_mlp_down_proj_weight5, alloc897) R.vm.kill_object(alloc896) R.vm.kill_object(model_layers_15_mlp_down_proj_weight5) model_layers_16_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[101] gv1014: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc898: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv1014, R.dtype("float16")) gv1015: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc899: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv1015, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc897, alloc894, model_layers_16_input_layernorm_weight5, alloc898, alloc899) R.vm.kill_object(alloc894) R.vm.kill_object(alloc897) R.vm.kill_object(model_layers_16_input_layernorm_weight5) lv206: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc898, alloc899 model_layers_16_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[97] gv1016: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc900: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv1016, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc898, model_layers_16_self_attn_qkv_proj_weight5, alloc900) R.vm.kill_object(alloc898) R.vm.kill_object(model_layers_16_self_attn_qkv_proj_weight5) gv1017: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape352: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc900, gv1017, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc900) gv1018: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape353: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape352, gv1018, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape352) gv1019: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc901: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1019, R.dtype("float16")) _724: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(16), R.prim_value(T.float32(1.0)), reshape353, alloc901) R.vm.kill_object(reshape353) gv1020: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape354: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc901, gv1020, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc901) gv1021: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape355: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape354, gv1021, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape354) model_layers_16_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[98] gv1022: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc902: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv1022, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape355, model_layers_16_self_attn_o_proj_weight5, alloc902) R.vm.kill_object(reshape355) R.vm.kill_object(model_layers_16_self_attn_o_proj_weight5) model_layers_16_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[102] gv1023: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc903: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1023, R.dtype("float16")) gv1024: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc904: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv1024, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc902, alloc899, model_layers_16_post_attention_layernorm_weight5, alloc903, alloc904) R.vm.kill_object(alloc899) R.vm.kill_object(alloc902) R.vm.kill_object(model_layers_16_post_attention_layernorm_weight5) lv208: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc903, alloc904 model_layers_16_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[99] gv1025: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc905: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv1025, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc903, model_layers_16_mlp_gate_up_proj_weight5, alloc905) R.vm.kill_object(alloc903) R.vm.kill_object(model_layers_16_mlp_gate_up_proj_weight5) gv1026: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc906: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv1026, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc905, alloc906) R.vm.kill_object(alloc905) model_layers_16_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[100] gv1027: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc907: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1027, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc906, model_layers_16_mlp_down_proj_weight5, alloc907) R.vm.kill_object(alloc906) R.vm.kill_object(model_layers_16_mlp_down_proj_weight5) model_layers_17_input_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[107] gv1028: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc908: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv1028, R.dtype("float16")) gv1029: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc909: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv1029, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc907, alloc904, model_layers_17_input_layernorm_weight5, alloc908, alloc909) R.vm.kill_object(alloc904) R.vm.kill_object(alloc907) R.vm.kill_object(model_layers_17_input_layernorm_weight5) lv210: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc908, alloc909 model_layers_17_self_attn_qkv_proj_weight5: R.Tensor((2560, 2048), dtype="float16") = packed_params[103] gv1030: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc910: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv1030, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc908, model_layers_17_self_attn_qkv_proj_weight5, alloc910) R.vm.kill_object(alloc908) R.vm.kill_object(model_layers_17_self_attn_qkv_proj_weight5) gv1031: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape356: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc910, gv1031, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc910) gv1032: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape357: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape356, gv1032, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape356) gv1033: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc911: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1033, R.dtype("float16")) _732: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(17), R.prim_value(T.float32(1.0)), reshape357, alloc911) R.vm.kill_object(reshape357) gv1034: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape358: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc911, gv1034, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc911) gv1035: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape359: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape358, gv1035, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape358) model_layers_17_self_attn_o_proj_weight5: R.Tensor((2048, 2048), dtype="float16") = packed_params[104] gv1036: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc912: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv1036, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape359, model_layers_17_self_attn_o_proj_weight5, alloc912) R.vm.kill_object(reshape359) R.vm.kill_object(model_layers_17_self_attn_o_proj_weight5) model_layers_17_post_attention_layernorm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[108] gv1037: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc913: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1037, R.dtype("float16")) gv1038: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc914: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage27, R.prim_value(0), gv1038, R.dtype("float16")) R.vm.kill_object(storage27) cls.gemma_q0f16_fuse_add_norm_prefill(alloc912, alloc909, model_layers_17_post_attention_layernorm_weight5, alloc913, alloc914) R.vm.kill_object(alloc909) R.vm.kill_object(alloc912) R.vm.kill_object(model_layers_17_post_attention_layernorm_weight5) lv212: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc913, alloc914 model_layers_17_mlp_gate_up_proj_weight5: R.Tensor((32768, 2048), dtype="float16") = packed_params[105] gv1039: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc915: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv1039, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc913, model_layers_17_mlp_gate_up_proj_weight5, alloc915) R.vm.kill_object(alloc913) R.vm.kill_object(model_layers_17_mlp_gate_up_proj_weight5) gv1040: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc916: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv1040, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc915, alloc916) R.vm.kill_object(alloc915) model_layers_17_mlp_down_proj_weight5: R.Tensor((2048, 16384), dtype="float16") = packed_params[106] gv1041: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc917: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage26, R.prim_value(0), gv1041, R.dtype("float16")) R.vm.kill_object(storage26) cls.gemma_q0f16_NT_matmul8(alloc916, model_layers_17_mlp_down_proj_weight5, alloc917) R.vm.kill_object(alloc916) R.vm.kill_object(model_layers_17_mlp_down_proj_weight5) model_norm_weight5: R.Tensor((2048,), dtype="float16") = packed_params[109] gv1042: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc918: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage29, R.prim_value(0), gv1042, R.dtype("float16")) R.vm.kill_object(storage29) gv1043: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc919: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage28, R.prim_value(0), gv1043, R.dtype("float16")) R.vm.kill_object(storage28) cls.gemma_q0f16_fuse_add_norm_prefill(alloc917, alloc914, model_norm_weight5, alloc918, alloc919) R.vm.kill_object(alloc914) R.vm.kill_object(alloc917) R.vm.kill_object(model_norm_weight5) lv214: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc918, alloc919 R.vm.kill_object(alloc919) model_embed_tokens_weight5: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[0] gv1044: R.Shape(ndim=1) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(3), sinfo_args=(R.Shape(ndim=1),)) storage30: R.Object = R.vm.alloc_storage(gv1044, R.prim_value(0), R.dtype("uint8"), R.str("global")) gv1045: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(1), sinfo_args=(R.Shape(ndim=3),)) alloc920: R.Tensor(dtype="float32", ndim=3) = R.vm.alloc_tensor(storage30, R.prim_value(0), gv1045, R.dtype("float32")) R.vm.kill_object(storage30) cls.gemma_q0f16_NT_matmul9(alloc918, model_embed_tokens_weight5, alloc920) R.vm.kill_object(alloc918) R.vm.kill_object(model_embed_tokens_weight5) gv5: R.Tuple(R.Tensor(dtype="float32", ndim=3), R.Object) = alloc920, paged_kv_cache R.vm.kill_object(alloc920) gv1046: R.Tensor(dtype="float32", ndim=3) = gv5[0] R.call_packed("vm.builtin.match_shape", gv1046, shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(3), R.prim_value(0), R.prim_value(3), R.prim_value(1), R.str("ErrorContext(fn=batch_verify, loc=return, annotation=R.Tuple(R.Tensor((1, seq_len, vocab_size), dtype=\"float32\"), R.Object)) "), sinfo_args=(R.Tuple,)) return gv5 @R.function def create_tir_paged_kv_cache(max_batch_size_: R.Shape(["max_batch_size"]), max_total_seq_len_: R.Shape(["max_total_seq_len"]), prefill_chunk_size_: R.Shape(["prefill_chunk_size"]), page_size_: R.Shape(["page_size"]), support_sliding_window_: R.Shape(["support_sliding_window"])) -> R.Object: max_batch_size = T.int64() max_total_seq_len = T.int64() prefill_chunk_size = T.int64() page_size = T.int64() support_sliding_window = T.int64() R.func_attr({"relax.force_pure": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 8192, "total_seq_len": 8192}}) cls = Module shape_heap: R.Tensor(dtype="int64", ndim=1) = R.call_builtin_with_ctx("vm.builtin.alloc_shape_heap", (R.prim_value(6),), sinfo_args=(R.Tensor(dtype="int64", ndim=1),)) R.call_packed("vm.builtin.check_shape_info", max_batch_size_, R.prim_value(1), R.str("ErrorContext(fn=create_tir_paged_kv_cache, loc=param[0], param=max_batch_size_, annotation=R.Shape([max_batch_size])) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_shape_info", max_total_seq_len_, R.prim_value(1), R.str("ErrorContext(fn=create_tir_paged_kv_cache, loc=param[1], param=max_total_seq_len_, annotation=R.Shape([max_total_seq_len])) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_shape_info", prefill_chunk_size_, R.prim_value(1), R.str("ErrorContext(fn=create_tir_paged_kv_cache, loc=param[2], param=prefill_chunk_size_, annotation=R.Shape([prefill_chunk_size])) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_shape_info", page_size_, R.prim_value(1), R.str("ErrorContext(fn=create_tir_paged_kv_cache, loc=param[3], param=page_size_, annotation=R.Shape([page_size])) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_shape_info", support_sliding_window_, R.prim_value(1), R.str("ErrorContext(fn=create_tir_paged_kv_cache, loc=param[4], param=support_sliding_window_, annotation=R.Shape([support_sliding_window])) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", max_batch_size_, shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(0), R.str("ErrorContext(fn=create_tir_paged_kv_cache, loc=param[0], param=max_batch_size_, annotation=R.Shape([max_batch_size])) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", max_total_seq_len_, shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(1), R.str("ErrorContext(fn=create_tir_paged_kv_cache, loc=param[1], param=max_total_seq_len_, annotation=R.Shape([max_total_seq_len])) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", prefill_chunk_size_, shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(2), R.str("ErrorContext(fn=create_tir_paged_kv_cache, loc=param[2], param=prefill_chunk_size_, annotation=R.Shape([prefill_chunk_size])) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", page_size_, shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(3), R.str("ErrorContext(fn=create_tir_paged_kv_cache, loc=param[3], param=page_size_, annotation=R.Shape([page_size])) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", support_sliding_window_, shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(4), R.str("ErrorContext(fn=create_tir_paged_kv_cache, loc=param[4], param=support_sliding_window_, annotation=R.Shape([support_sliding_window])) "), sinfo_args=(R.Tuple,)) gv1047: R.Shape(ndim=5) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(5), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(1), R.prim_value(2), R.prim_value(1), R.prim_value(3), R.prim_value(1), R.prim_value(4), sinfo_args=(R.Shape(ndim=5),)) paged_kv_cache: R.Object = R.call_packed("vm.builtin.paged_attention_kv_cache_create_reduced", gv1047, R.shape([0, 18]), R.prim_value(8), R.prim_value(1), R.prim_value(256), R.prim_value(1), R.prim_value(1), R.prim_value(T.float32(10000.0)), R.const(0.0, "float16"), cls.gemma_q0f16_tir_kv_cache_transpose_append, cls.gemma_q0f16_batch_prefill_paged_kv, cls.gemma_q0f16_batch_decode_paged_kv, cls.gemma_q0f16_batch_prefill_paged_kv_sliding_window, cls.gemma_q0f16_batch_decode_paged_kv_sliding_window, cls.gemma_q0f16_batch_prefill_ragged_kv, cls.gemma_q0f16_merge_state_inplace, cls.gemma_q0f16_fused_rope, cls.gemma_q0f16_copy_single_page, cls.gemma_q0f16_tir_kv_cache_debug_get_kv, cls.gemma_q0f16_compact_kv_copy, cls.gemma_q0f16_batch_tree_attn, cls.gemma_q0f16_tree_attn_paged_kv, R.prim_value(0), R.prim_value(0), sinfo_args=(R.Object,)) return paged_kv_cache @R.function def decode(input_embed: R.Tensor((1, 1, 2048), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 2048), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"))) -> R.Tuple(R.Tensor((1, 1, "vocab_size"), dtype="float32"), R.Object): vocab_size = T.int64() R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.force_pure": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 8192, "total_seq_len": 8192}}) cls = Module shape_heap: R.Tensor(dtype="int64", ndim=1) = R.call_builtin_with_ctx("vm.builtin.alloc_shape_heap", (R.prim_value(3),), sinfo_args=(R.Tensor(dtype="int64", ndim=1),)) R.call_packed("vm.builtin.check_tensor_info", input_embed, R.prim_value(3), R.dtype("float16"), R.str("ErrorContext(fn=decode, loc=param[0], param=input_embed, annotation=R.Tensor((1, 1, 2048), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_tuple_info", packed_params, R.prim_value(110), R.str("ErrorContext(fn=decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 2048), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv785: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[0] R.call_packed("vm.builtin.check_tensor_info", gv785, R.prim_value(2), R.dtype("float16"), R.str("ErrorContext(fn=decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 2048), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", input_embed, shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(2048), R.str("ErrorContext(fn=decode, loc=param[0], param=input_embed, annotation=R.Tensor((1, 1, 2048), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv785, shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), R.str("ErrorContext(fn=decode, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 2048), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) cls.gemma_q0f16_shape_func4(shape_heap) storage21: R.Object = R.vm.alloc_storage(R.shape([4096]), R.prim_value(0), R.dtype("uint8"), R.str("global")) alloc555: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_multiply4(input_embed, alloc555) model_layers_0_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[5] storage22: R.Object = R.vm.alloc_storage(R.shape([32768]), R.prim_value(0), R.dtype("uint8"), R.str("global")) alloc556: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_rms_norm2(alloc555, model_layers_0_input_layernorm_weight2, alloc556) R.vm.kill_object(model_layers_0_input_layernorm_weight2) model_layers_0_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[1] storage23: R.Object = R.vm.alloc_storage(R.shape([65536]), R.prim_value(0), R.dtype("uint8"), R.str("global")) alloc557: R.Tensor((1, 1, 2560), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 2560]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul10(alloc556, model_layers_0_self_attn_qkv_proj_weight2, alloc557) R.vm.kill_object(alloc556) R.vm.kill_object(model_layers_0_self_attn_qkv_proj_weight2) lv54: R.Tensor((1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc557, R.shape([1, 10, 256]), sinfo_args=(R.Tensor((1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc557) alloc558: R.Tensor((1, 8, 256), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 8, 256]), R.dtype("float16")) _449: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), lv54, alloc558) R.vm.kill_object(lv54) lv55: R.Tensor((1, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc558, R.shape([1, 1, 2048]), sinfo_args=(R.Tensor((1, 1, 2048), dtype="float16"),)) R.vm.kill_object(alloc558) model_layers_0_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[2] alloc559: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul11(lv55, model_layers_0_self_attn_o_proj_weight2, alloc559) R.vm.kill_object(lv55) R.vm.kill_object(model_layers_0_self_attn_o_proj_weight2) model_layers_0_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[6] alloc560: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) storage24: R.Object = R.vm.alloc_storage(R.shape([32768]), R.prim_value(0), R.dtype("uint8"), R.str("global")) alloc561: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc559, alloc555, model_layers_0_post_attention_layernorm_weight2, alloc560, alloc561) R.vm.kill_object(alloc555) R.vm.kill_object(alloc559) R.vm.kill_object(model_layers_0_post_attention_layernorm_weight2) lv216: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc560, alloc561 model_layers_0_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[3] alloc562: R.Tensor((1, 1, 32768), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 32768]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul12(alloc560, model_layers_0_mlp_gate_up_proj_weight2, alloc562) R.vm.kill_object(alloc560) R.vm.kill_object(model_layers_0_mlp_gate_up_proj_weight2) alloc563: R.Tensor((1, 1, 16384), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 16384]), R.dtype("float16")) cls.gemma_q0f16_fused_split2_gelu_tanh2_multiply5(alloc562, alloc563) R.vm.kill_object(alloc562) model_layers_0_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[4] alloc564: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul13(alloc563, model_layers_0_mlp_down_proj_weight2, alloc564) R.vm.kill_object(alloc563) R.vm.kill_object(model_layers_0_mlp_down_proj_weight2) model_layers_1_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[11] alloc565: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc566: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc564, alloc561, model_layers_1_input_layernorm_weight2, alloc565, alloc566) R.vm.kill_object(alloc561) R.vm.kill_object(alloc564) R.vm.kill_object(model_layers_1_input_layernorm_weight2) lv218: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc565, alloc566 model_layers_1_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[7] alloc567: R.Tensor((1, 1, 2560), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2560]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul10(alloc565, model_layers_1_self_attn_qkv_proj_weight2, alloc567) R.vm.kill_object(alloc565) R.vm.kill_object(model_layers_1_self_attn_qkv_proj_weight2) lv57: R.Tensor((1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc567, R.shape([1, 10, 256]), sinfo_args=(R.Tensor((1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc567) alloc568: R.Tensor((1, 8, 256), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 8, 256]), R.dtype("float16")) _457: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), lv57, alloc568) R.vm.kill_object(lv57) lv58: R.Tensor((1, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc568, R.shape([1, 1, 2048]), sinfo_args=(R.Tensor((1, 1, 2048), dtype="float16"),)) R.vm.kill_object(alloc568) model_layers_1_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[8] alloc569: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul11(lv58, model_layers_1_self_attn_o_proj_weight2, alloc569) R.vm.kill_object(lv58) R.vm.kill_object(model_layers_1_self_attn_o_proj_weight2) model_layers_1_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[12] alloc570: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc571: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc569, alloc566, model_layers_1_post_attention_layernorm_weight2, alloc570, alloc571) R.vm.kill_object(alloc566) R.vm.kill_object(alloc569) R.vm.kill_object(model_layers_1_post_attention_layernorm_weight2) lv220: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc570, alloc571 model_layers_1_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[9] alloc572: R.Tensor((1, 1, 32768), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 32768]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul12(alloc570, model_layers_1_mlp_gate_up_proj_weight2, alloc572) R.vm.kill_object(alloc570) R.vm.kill_object(model_layers_1_mlp_gate_up_proj_weight2) alloc573: R.Tensor((1, 1, 16384), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 16384]), R.dtype("float16")) cls.gemma_q0f16_fused_split2_gelu_tanh2_multiply5(alloc572, alloc573) R.vm.kill_object(alloc572) model_layers_1_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[10] alloc574: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul13(alloc573, model_layers_1_mlp_down_proj_weight2, alloc574) R.vm.kill_object(alloc573) R.vm.kill_object(model_layers_1_mlp_down_proj_weight2) model_layers_2_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[17] alloc575: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc576: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc574, alloc571, model_layers_2_input_layernorm_weight2, alloc575, alloc576) R.vm.kill_object(alloc571) R.vm.kill_object(alloc574) R.vm.kill_object(model_layers_2_input_layernorm_weight2) lv222: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc575, alloc576 model_layers_2_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[13] alloc577: R.Tensor((1, 1, 2560), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2560]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul10(alloc575, model_layers_2_self_attn_qkv_proj_weight2, alloc577) R.vm.kill_object(alloc575) R.vm.kill_object(model_layers_2_self_attn_qkv_proj_weight2) lv60: R.Tensor((1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc577, R.shape([1, 10, 256]), sinfo_args=(R.Tensor((1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc577) alloc578: R.Tensor((1, 8, 256), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 8, 256]), R.dtype("float16")) _465: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), lv60, alloc578) R.vm.kill_object(lv60) lv61: R.Tensor((1, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc578, R.shape([1, 1, 2048]), sinfo_args=(R.Tensor((1, 1, 2048), dtype="float16"),)) R.vm.kill_object(alloc578) model_layers_2_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[14] alloc579: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul11(lv61, model_layers_2_self_attn_o_proj_weight2, alloc579) R.vm.kill_object(lv61) R.vm.kill_object(model_layers_2_self_attn_o_proj_weight2) model_layers_2_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[18] alloc580: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc581: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc579, alloc576, model_layers_2_post_attention_layernorm_weight2, alloc580, alloc581) R.vm.kill_object(alloc576) R.vm.kill_object(alloc579) R.vm.kill_object(model_layers_2_post_attention_layernorm_weight2) lv224: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc580, alloc581 model_layers_2_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[15] alloc582: R.Tensor((1, 1, 32768), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 32768]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul12(alloc580, model_layers_2_mlp_gate_up_proj_weight2, alloc582) R.vm.kill_object(alloc580) R.vm.kill_object(model_layers_2_mlp_gate_up_proj_weight2) alloc583: R.Tensor((1, 1, 16384), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 16384]), R.dtype("float16")) cls.gemma_q0f16_fused_split2_gelu_tanh2_multiply5(alloc582, alloc583) R.vm.kill_object(alloc582) model_layers_2_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[16] alloc584: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul13(alloc583, model_layers_2_mlp_down_proj_weight2, alloc584) R.vm.kill_object(alloc583) R.vm.kill_object(model_layers_2_mlp_down_proj_weight2) model_layers_3_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[23] alloc585: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc586: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc584, alloc581, model_layers_3_input_layernorm_weight2, alloc585, alloc586) R.vm.kill_object(alloc581) R.vm.kill_object(alloc584) R.vm.kill_object(model_layers_3_input_layernorm_weight2) lv226: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc585, alloc586 model_layers_3_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[19] alloc587: R.Tensor((1, 1, 2560), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2560]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul10(alloc585, model_layers_3_self_attn_qkv_proj_weight2, alloc587) R.vm.kill_object(alloc585) R.vm.kill_object(model_layers_3_self_attn_qkv_proj_weight2) lv63: R.Tensor((1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc587, R.shape([1, 10, 256]), sinfo_args=(R.Tensor((1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc587) alloc588: R.Tensor((1, 8, 256), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 8, 256]), R.dtype("float16")) _473: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), lv63, alloc588) R.vm.kill_object(lv63) lv64: R.Tensor((1, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc588, R.shape([1, 1, 2048]), sinfo_args=(R.Tensor((1, 1, 2048), dtype="float16"),)) R.vm.kill_object(alloc588) model_layers_3_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[20] alloc589: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul11(lv64, model_layers_3_self_attn_o_proj_weight2, alloc589) R.vm.kill_object(lv64) R.vm.kill_object(model_layers_3_self_attn_o_proj_weight2) model_layers_3_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[24] alloc590: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc591: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc589, alloc586, model_layers_3_post_attention_layernorm_weight2, alloc590, alloc591) R.vm.kill_object(alloc586) R.vm.kill_object(alloc589) R.vm.kill_object(model_layers_3_post_attention_layernorm_weight2) lv228: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc590, alloc591 model_layers_3_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[21] alloc592: R.Tensor((1, 1, 32768), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 32768]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul12(alloc590, model_layers_3_mlp_gate_up_proj_weight2, alloc592) R.vm.kill_object(alloc590) R.vm.kill_object(model_layers_3_mlp_gate_up_proj_weight2) alloc593: R.Tensor((1, 1, 16384), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 16384]), R.dtype("float16")) cls.gemma_q0f16_fused_split2_gelu_tanh2_multiply5(alloc592, alloc593) R.vm.kill_object(alloc592) model_layers_3_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[22] alloc594: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul13(alloc593, model_layers_3_mlp_down_proj_weight2, alloc594) R.vm.kill_object(alloc593) R.vm.kill_object(model_layers_3_mlp_down_proj_weight2) model_layers_4_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[29] alloc595: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc596: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc594, alloc591, model_layers_4_input_layernorm_weight2, alloc595, alloc596) R.vm.kill_object(alloc591) R.vm.kill_object(alloc594) R.vm.kill_object(model_layers_4_input_layernorm_weight2) lv230: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc595, alloc596 model_layers_4_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[25] alloc597: R.Tensor((1, 1, 2560), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2560]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul10(alloc595, model_layers_4_self_attn_qkv_proj_weight2, alloc597) R.vm.kill_object(alloc595) R.vm.kill_object(model_layers_4_self_attn_qkv_proj_weight2) lv66: R.Tensor((1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc597, R.shape([1, 10, 256]), sinfo_args=(R.Tensor((1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc597) alloc598: R.Tensor((1, 8, 256), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 8, 256]), R.dtype("float16")) _481: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), lv66, alloc598) R.vm.kill_object(lv66) lv67: R.Tensor((1, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc598, R.shape([1, 1, 2048]), sinfo_args=(R.Tensor((1, 1, 2048), dtype="float16"),)) R.vm.kill_object(alloc598) model_layers_4_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[26] alloc599: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul11(lv67, model_layers_4_self_attn_o_proj_weight2, alloc599) R.vm.kill_object(lv67) R.vm.kill_object(model_layers_4_self_attn_o_proj_weight2) model_layers_4_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[30] alloc600: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc601: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc599, alloc596, model_layers_4_post_attention_layernorm_weight2, alloc600, alloc601) R.vm.kill_object(alloc596) R.vm.kill_object(alloc599) R.vm.kill_object(model_layers_4_post_attention_layernorm_weight2) lv232: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc600, alloc601 model_layers_4_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[27] alloc602: R.Tensor((1, 1, 32768), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 32768]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul12(alloc600, model_layers_4_mlp_gate_up_proj_weight2, alloc602) R.vm.kill_object(alloc600) R.vm.kill_object(model_layers_4_mlp_gate_up_proj_weight2) alloc603: R.Tensor((1, 1, 16384), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 16384]), R.dtype("float16")) cls.gemma_q0f16_fused_split2_gelu_tanh2_multiply5(alloc602, alloc603) R.vm.kill_object(alloc602) model_layers_4_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[28] alloc604: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul13(alloc603, model_layers_4_mlp_down_proj_weight2, alloc604) R.vm.kill_object(alloc603) R.vm.kill_object(model_layers_4_mlp_down_proj_weight2) model_layers_5_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[35] alloc605: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc606: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc604, alloc601, model_layers_5_input_layernorm_weight2, alloc605, alloc606) R.vm.kill_object(alloc601) R.vm.kill_object(alloc604) R.vm.kill_object(model_layers_5_input_layernorm_weight2) lv234: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc605, alloc606 model_layers_5_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[31] alloc607: R.Tensor((1, 1, 2560), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2560]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul10(alloc605, model_layers_5_self_attn_qkv_proj_weight2, alloc607) R.vm.kill_object(alloc605) R.vm.kill_object(model_layers_5_self_attn_qkv_proj_weight2) lv69: R.Tensor((1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc607, R.shape([1, 10, 256]), sinfo_args=(R.Tensor((1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc607) alloc608: R.Tensor((1, 8, 256), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 8, 256]), R.dtype("float16")) _489: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), lv69, alloc608) R.vm.kill_object(lv69) lv70: R.Tensor((1, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc608, R.shape([1, 1, 2048]), sinfo_args=(R.Tensor((1, 1, 2048), dtype="float16"),)) R.vm.kill_object(alloc608) model_layers_5_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[32] alloc609: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul11(lv70, model_layers_5_self_attn_o_proj_weight2, alloc609) R.vm.kill_object(lv70) R.vm.kill_object(model_layers_5_self_attn_o_proj_weight2) model_layers_5_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[36] alloc610: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc611: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc609, alloc606, model_layers_5_post_attention_layernorm_weight2, alloc610, alloc611) R.vm.kill_object(alloc606) R.vm.kill_object(alloc609) R.vm.kill_object(model_layers_5_post_attention_layernorm_weight2) lv236: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc610, alloc611 model_layers_5_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[33] alloc612: R.Tensor((1, 1, 32768), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 32768]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul12(alloc610, model_layers_5_mlp_gate_up_proj_weight2, alloc612) R.vm.kill_object(alloc610) R.vm.kill_object(model_layers_5_mlp_gate_up_proj_weight2) alloc613: R.Tensor((1, 1, 16384), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 16384]), R.dtype("float16")) cls.gemma_q0f16_fused_split2_gelu_tanh2_multiply5(alloc612, alloc613) R.vm.kill_object(alloc612) model_layers_5_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[34] alloc614: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul13(alloc613, model_layers_5_mlp_down_proj_weight2, alloc614) R.vm.kill_object(alloc613) R.vm.kill_object(model_layers_5_mlp_down_proj_weight2) model_layers_6_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[41] alloc615: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc616: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc614, alloc611, model_layers_6_input_layernorm_weight2, alloc615, alloc616) R.vm.kill_object(alloc611) R.vm.kill_object(alloc614) R.vm.kill_object(model_layers_6_input_layernorm_weight2) lv238: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc615, alloc616 model_layers_6_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[37] alloc617: R.Tensor((1, 1, 2560), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2560]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul10(alloc615, model_layers_6_self_attn_qkv_proj_weight2, alloc617) R.vm.kill_object(alloc615) R.vm.kill_object(model_layers_6_self_attn_qkv_proj_weight2) lv72: R.Tensor((1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc617, R.shape([1, 10, 256]), sinfo_args=(R.Tensor((1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc617) alloc618: R.Tensor((1, 8, 256), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 8, 256]), R.dtype("float16")) _497: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(6), R.prim_value(T.float32(1.0)), lv72, alloc618) R.vm.kill_object(lv72) lv73: R.Tensor((1, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc618, R.shape([1, 1, 2048]), sinfo_args=(R.Tensor((1, 1, 2048), dtype="float16"),)) R.vm.kill_object(alloc618) model_layers_6_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[38] alloc619: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul11(lv73, model_layers_6_self_attn_o_proj_weight2, alloc619) R.vm.kill_object(lv73) R.vm.kill_object(model_layers_6_self_attn_o_proj_weight2) model_layers_6_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[42] alloc620: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc621: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc619, alloc616, model_layers_6_post_attention_layernorm_weight2, alloc620, alloc621) R.vm.kill_object(alloc616) R.vm.kill_object(alloc619) R.vm.kill_object(model_layers_6_post_attention_layernorm_weight2) lv240: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc620, alloc621 model_layers_6_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[39] alloc622: R.Tensor((1, 1, 32768), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 32768]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul12(alloc620, model_layers_6_mlp_gate_up_proj_weight2, alloc622) R.vm.kill_object(alloc620) R.vm.kill_object(model_layers_6_mlp_gate_up_proj_weight2) alloc623: R.Tensor((1, 1, 16384), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 16384]), R.dtype("float16")) cls.gemma_q0f16_fused_split2_gelu_tanh2_multiply5(alloc622, alloc623) R.vm.kill_object(alloc622) model_layers_6_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[40] alloc624: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul13(alloc623, model_layers_6_mlp_down_proj_weight2, alloc624) R.vm.kill_object(alloc623) R.vm.kill_object(model_layers_6_mlp_down_proj_weight2) model_layers_7_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[47] alloc625: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc626: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc624, alloc621, model_layers_7_input_layernorm_weight2, alloc625, alloc626) R.vm.kill_object(alloc621) R.vm.kill_object(alloc624) R.vm.kill_object(model_layers_7_input_layernorm_weight2) lv242: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc625, alloc626 model_layers_7_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[43] alloc627: R.Tensor((1, 1, 2560), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2560]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul10(alloc625, model_layers_7_self_attn_qkv_proj_weight2, alloc627) R.vm.kill_object(alloc625) R.vm.kill_object(model_layers_7_self_attn_qkv_proj_weight2) lv75: R.Tensor((1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc627, R.shape([1, 10, 256]), sinfo_args=(R.Tensor((1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc627) alloc628: R.Tensor((1, 8, 256), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 8, 256]), R.dtype("float16")) _505: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(7), R.prim_value(T.float32(1.0)), lv75, alloc628) R.vm.kill_object(lv75) lv76: R.Tensor((1, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc628, R.shape([1, 1, 2048]), sinfo_args=(R.Tensor((1, 1, 2048), dtype="float16"),)) R.vm.kill_object(alloc628) model_layers_7_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[44] alloc629: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul11(lv76, model_layers_7_self_attn_o_proj_weight2, alloc629) R.vm.kill_object(lv76) R.vm.kill_object(model_layers_7_self_attn_o_proj_weight2) model_layers_7_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[48] alloc630: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc631: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc629, alloc626, model_layers_7_post_attention_layernorm_weight2, alloc630, alloc631) R.vm.kill_object(alloc626) R.vm.kill_object(alloc629) R.vm.kill_object(model_layers_7_post_attention_layernorm_weight2) lv244: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc630, alloc631 model_layers_7_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[45] alloc632: R.Tensor((1, 1, 32768), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 32768]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul12(alloc630, model_layers_7_mlp_gate_up_proj_weight2, alloc632) R.vm.kill_object(alloc630) R.vm.kill_object(model_layers_7_mlp_gate_up_proj_weight2) alloc633: R.Tensor((1, 1, 16384), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 16384]), R.dtype("float16")) cls.gemma_q0f16_fused_split2_gelu_tanh2_multiply5(alloc632, alloc633) R.vm.kill_object(alloc632) model_layers_7_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[46] alloc634: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul13(alloc633, model_layers_7_mlp_down_proj_weight2, alloc634) R.vm.kill_object(alloc633) R.vm.kill_object(model_layers_7_mlp_down_proj_weight2) model_layers_8_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[53] alloc635: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc636: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc634, alloc631, model_layers_8_input_layernorm_weight2, alloc635, alloc636) R.vm.kill_object(alloc631) R.vm.kill_object(alloc634) R.vm.kill_object(model_layers_8_input_layernorm_weight2) lv246: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc635, alloc636 model_layers_8_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[49] alloc637: R.Tensor((1, 1, 2560), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2560]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul10(alloc635, model_layers_8_self_attn_qkv_proj_weight2, alloc637) R.vm.kill_object(alloc635) R.vm.kill_object(model_layers_8_self_attn_qkv_proj_weight2) lv78: R.Tensor((1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc637, R.shape([1, 10, 256]), sinfo_args=(R.Tensor((1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc637) alloc638: R.Tensor((1, 8, 256), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 8, 256]), R.dtype("float16")) _513: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(8), R.prim_value(T.float32(1.0)), lv78, alloc638) R.vm.kill_object(lv78) lv79: R.Tensor((1, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc638, R.shape([1, 1, 2048]), sinfo_args=(R.Tensor((1, 1, 2048), dtype="float16"),)) R.vm.kill_object(alloc638) model_layers_8_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[50] alloc639: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul11(lv79, model_layers_8_self_attn_o_proj_weight2, alloc639) R.vm.kill_object(lv79) R.vm.kill_object(model_layers_8_self_attn_o_proj_weight2) model_layers_8_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[54] alloc640: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc641: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc639, alloc636, model_layers_8_post_attention_layernorm_weight2, alloc640, alloc641) R.vm.kill_object(alloc636) R.vm.kill_object(alloc639) R.vm.kill_object(model_layers_8_post_attention_layernorm_weight2) lv248: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc640, alloc641 model_layers_8_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[51] alloc642: R.Tensor((1, 1, 32768), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 32768]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul12(alloc640, model_layers_8_mlp_gate_up_proj_weight2, alloc642) R.vm.kill_object(alloc640) R.vm.kill_object(model_layers_8_mlp_gate_up_proj_weight2) alloc643: R.Tensor((1, 1, 16384), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 16384]), R.dtype("float16")) cls.gemma_q0f16_fused_split2_gelu_tanh2_multiply5(alloc642, alloc643) R.vm.kill_object(alloc642) model_layers_8_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[52] alloc644: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul13(alloc643, model_layers_8_mlp_down_proj_weight2, alloc644) R.vm.kill_object(alloc643) R.vm.kill_object(model_layers_8_mlp_down_proj_weight2) model_layers_9_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[59] alloc645: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc646: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc644, alloc641, model_layers_9_input_layernorm_weight2, alloc645, alloc646) R.vm.kill_object(alloc641) R.vm.kill_object(alloc644) R.vm.kill_object(model_layers_9_input_layernorm_weight2) lv250: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc645, alloc646 model_layers_9_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[55] alloc647: R.Tensor((1, 1, 2560), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2560]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul10(alloc645, model_layers_9_self_attn_qkv_proj_weight2, alloc647) R.vm.kill_object(alloc645) R.vm.kill_object(model_layers_9_self_attn_qkv_proj_weight2) lv81: R.Tensor((1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc647, R.shape([1, 10, 256]), sinfo_args=(R.Tensor((1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc647) alloc648: R.Tensor((1, 8, 256), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 8, 256]), R.dtype("float16")) _521: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(9), R.prim_value(T.float32(1.0)), lv81, alloc648) R.vm.kill_object(lv81) lv82: R.Tensor((1, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc648, R.shape([1, 1, 2048]), sinfo_args=(R.Tensor((1, 1, 2048), dtype="float16"),)) R.vm.kill_object(alloc648) model_layers_9_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[56] alloc649: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul11(lv82, model_layers_9_self_attn_o_proj_weight2, alloc649) R.vm.kill_object(lv82) R.vm.kill_object(model_layers_9_self_attn_o_proj_weight2) model_layers_9_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[60] alloc650: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc651: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc649, alloc646, model_layers_9_post_attention_layernorm_weight2, alloc650, alloc651) R.vm.kill_object(alloc646) R.vm.kill_object(alloc649) R.vm.kill_object(model_layers_9_post_attention_layernorm_weight2) lv252: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc650, alloc651 model_layers_9_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[57] alloc652: R.Tensor((1, 1, 32768), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 32768]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul12(alloc650, model_layers_9_mlp_gate_up_proj_weight2, alloc652) R.vm.kill_object(alloc650) R.vm.kill_object(model_layers_9_mlp_gate_up_proj_weight2) alloc653: R.Tensor((1, 1, 16384), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 16384]), R.dtype("float16")) cls.gemma_q0f16_fused_split2_gelu_tanh2_multiply5(alloc652, alloc653) R.vm.kill_object(alloc652) model_layers_9_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[58] alloc654: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul13(alloc653, model_layers_9_mlp_down_proj_weight2, alloc654) R.vm.kill_object(alloc653) R.vm.kill_object(model_layers_9_mlp_down_proj_weight2) model_layers_10_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[65] alloc655: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc656: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc654, alloc651, model_layers_10_input_layernorm_weight2, alloc655, alloc656) R.vm.kill_object(alloc651) R.vm.kill_object(alloc654) R.vm.kill_object(model_layers_10_input_layernorm_weight2) lv254: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc655, alloc656 model_layers_10_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[61] alloc657: R.Tensor((1, 1, 2560), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2560]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul10(alloc655, model_layers_10_self_attn_qkv_proj_weight2, alloc657) R.vm.kill_object(alloc655) R.vm.kill_object(model_layers_10_self_attn_qkv_proj_weight2) lv84: R.Tensor((1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc657, R.shape([1, 10, 256]), sinfo_args=(R.Tensor((1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc657) alloc658: R.Tensor((1, 8, 256), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 8, 256]), R.dtype("float16")) _529: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(10), R.prim_value(T.float32(1.0)), lv84, alloc658) R.vm.kill_object(lv84) lv85: R.Tensor((1, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc658, R.shape([1, 1, 2048]), sinfo_args=(R.Tensor((1, 1, 2048), dtype="float16"),)) R.vm.kill_object(alloc658) model_layers_10_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[62] alloc659: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul11(lv85, model_layers_10_self_attn_o_proj_weight2, alloc659) R.vm.kill_object(lv85) R.vm.kill_object(model_layers_10_self_attn_o_proj_weight2) model_layers_10_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[66] alloc660: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc661: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc659, alloc656, model_layers_10_post_attention_layernorm_weight2, alloc660, alloc661) R.vm.kill_object(alloc656) R.vm.kill_object(alloc659) R.vm.kill_object(model_layers_10_post_attention_layernorm_weight2) lv256: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc660, alloc661 model_layers_10_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[63] alloc662: R.Tensor((1, 1, 32768), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 32768]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul12(alloc660, model_layers_10_mlp_gate_up_proj_weight2, alloc662) R.vm.kill_object(alloc660) R.vm.kill_object(model_layers_10_mlp_gate_up_proj_weight2) alloc663: R.Tensor((1, 1, 16384), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 16384]), R.dtype("float16")) cls.gemma_q0f16_fused_split2_gelu_tanh2_multiply5(alloc662, alloc663) R.vm.kill_object(alloc662) model_layers_10_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[64] alloc664: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul13(alloc663, model_layers_10_mlp_down_proj_weight2, alloc664) R.vm.kill_object(alloc663) R.vm.kill_object(model_layers_10_mlp_down_proj_weight2) model_layers_11_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[71] alloc665: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc666: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc664, alloc661, model_layers_11_input_layernorm_weight2, alloc665, alloc666) R.vm.kill_object(alloc661) R.vm.kill_object(alloc664) R.vm.kill_object(model_layers_11_input_layernorm_weight2) lv258: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc665, alloc666 model_layers_11_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[67] alloc667: R.Tensor((1, 1, 2560), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2560]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul10(alloc665, model_layers_11_self_attn_qkv_proj_weight2, alloc667) R.vm.kill_object(alloc665) R.vm.kill_object(model_layers_11_self_attn_qkv_proj_weight2) lv87: R.Tensor((1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc667, R.shape([1, 10, 256]), sinfo_args=(R.Tensor((1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc667) alloc668: R.Tensor((1, 8, 256), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 8, 256]), R.dtype("float16")) _537: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(11), R.prim_value(T.float32(1.0)), lv87, alloc668) R.vm.kill_object(lv87) lv88: R.Tensor((1, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc668, R.shape([1, 1, 2048]), sinfo_args=(R.Tensor((1, 1, 2048), dtype="float16"),)) R.vm.kill_object(alloc668) model_layers_11_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[68] alloc669: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul11(lv88, model_layers_11_self_attn_o_proj_weight2, alloc669) R.vm.kill_object(lv88) R.vm.kill_object(model_layers_11_self_attn_o_proj_weight2) model_layers_11_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[72] alloc670: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc671: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc669, alloc666, model_layers_11_post_attention_layernorm_weight2, alloc670, alloc671) R.vm.kill_object(alloc666) R.vm.kill_object(alloc669) R.vm.kill_object(model_layers_11_post_attention_layernorm_weight2) lv260: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc670, alloc671 model_layers_11_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[69] alloc672: R.Tensor((1, 1, 32768), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 32768]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul12(alloc670, model_layers_11_mlp_gate_up_proj_weight2, alloc672) R.vm.kill_object(alloc670) R.vm.kill_object(model_layers_11_mlp_gate_up_proj_weight2) alloc673: R.Tensor((1, 1, 16384), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 16384]), R.dtype("float16")) cls.gemma_q0f16_fused_split2_gelu_tanh2_multiply5(alloc672, alloc673) R.vm.kill_object(alloc672) model_layers_11_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[70] alloc674: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul13(alloc673, model_layers_11_mlp_down_proj_weight2, alloc674) R.vm.kill_object(alloc673) R.vm.kill_object(model_layers_11_mlp_down_proj_weight2) model_layers_12_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[77] alloc675: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc676: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc674, alloc671, model_layers_12_input_layernorm_weight2, alloc675, alloc676) R.vm.kill_object(alloc671) R.vm.kill_object(alloc674) R.vm.kill_object(model_layers_12_input_layernorm_weight2) lv262: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc675, alloc676 model_layers_12_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[73] alloc677: R.Tensor((1, 1, 2560), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2560]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul10(alloc675, model_layers_12_self_attn_qkv_proj_weight2, alloc677) R.vm.kill_object(alloc675) R.vm.kill_object(model_layers_12_self_attn_qkv_proj_weight2) lv90: R.Tensor((1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc677, R.shape([1, 10, 256]), sinfo_args=(R.Tensor((1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc677) alloc678: R.Tensor((1, 8, 256), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 8, 256]), R.dtype("float16")) _545: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(12), R.prim_value(T.float32(1.0)), lv90, alloc678) R.vm.kill_object(lv90) lv91: R.Tensor((1, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc678, R.shape([1, 1, 2048]), sinfo_args=(R.Tensor((1, 1, 2048), dtype="float16"),)) R.vm.kill_object(alloc678) model_layers_12_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[74] alloc679: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul11(lv91, model_layers_12_self_attn_o_proj_weight2, alloc679) R.vm.kill_object(lv91) R.vm.kill_object(model_layers_12_self_attn_o_proj_weight2) model_layers_12_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[78] alloc680: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc681: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc679, alloc676, model_layers_12_post_attention_layernorm_weight2, alloc680, alloc681) R.vm.kill_object(alloc676) R.vm.kill_object(alloc679) R.vm.kill_object(model_layers_12_post_attention_layernorm_weight2) lv264: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc680, alloc681 model_layers_12_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[75] alloc682: R.Tensor((1, 1, 32768), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 32768]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul12(alloc680, model_layers_12_mlp_gate_up_proj_weight2, alloc682) R.vm.kill_object(alloc680) R.vm.kill_object(model_layers_12_mlp_gate_up_proj_weight2) alloc683: R.Tensor((1, 1, 16384), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 16384]), R.dtype("float16")) cls.gemma_q0f16_fused_split2_gelu_tanh2_multiply5(alloc682, alloc683) R.vm.kill_object(alloc682) model_layers_12_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[76] alloc684: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul13(alloc683, model_layers_12_mlp_down_proj_weight2, alloc684) R.vm.kill_object(alloc683) R.vm.kill_object(model_layers_12_mlp_down_proj_weight2) model_layers_13_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[83] alloc685: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc686: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc684, alloc681, model_layers_13_input_layernorm_weight2, alloc685, alloc686) R.vm.kill_object(alloc681) R.vm.kill_object(alloc684) R.vm.kill_object(model_layers_13_input_layernorm_weight2) lv266: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc685, alloc686 model_layers_13_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[79] alloc687: R.Tensor((1, 1, 2560), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2560]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul10(alloc685, model_layers_13_self_attn_qkv_proj_weight2, alloc687) R.vm.kill_object(alloc685) R.vm.kill_object(model_layers_13_self_attn_qkv_proj_weight2) lv93: R.Tensor((1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc687, R.shape([1, 10, 256]), sinfo_args=(R.Tensor((1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc687) alloc688: R.Tensor((1, 8, 256), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 8, 256]), R.dtype("float16")) _553: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(13), R.prim_value(T.float32(1.0)), lv93, alloc688) R.vm.kill_object(lv93) lv94: R.Tensor((1, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc688, R.shape([1, 1, 2048]), sinfo_args=(R.Tensor((1, 1, 2048), dtype="float16"),)) R.vm.kill_object(alloc688) model_layers_13_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[80] alloc689: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul11(lv94, model_layers_13_self_attn_o_proj_weight2, alloc689) R.vm.kill_object(lv94) R.vm.kill_object(model_layers_13_self_attn_o_proj_weight2) model_layers_13_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[84] alloc690: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc691: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc689, alloc686, model_layers_13_post_attention_layernorm_weight2, alloc690, alloc691) R.vm.kill_object(alloc686) R.vm.kill_object(alloc689) R.vm.kill_object(model_layers_13_post_attention_layernorm_weight2) lv268: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc690, alloc691 model_layers_13_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[81] alloc692: R.Tensor((1, 1, 32768), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 32768]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul12(alloc690, model_layers_13_mlp_gate_up_proj_weight2, alloc692) R.vm.kill_object(alloc690) R.vm.kill_object(model_layers_13_mlp_gate_up_proj_weight2) alloc693: R.Tensor((1, 1, 16384), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 16384]), R.dtype("float16")) cls.gemma_q0f16_fused_split2_gelu_tanh2_multiply5(alloc692, alloc693) R.vm.kill_object(alloc692) model_layers_13_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[82] alloc694: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul13(alloc693, model_layers_13_mlp_down_proj_weight2, alloc694) R.vm.kill_object(alloc693) R.vm.kill_object(model_layers_13_mlp_down_proj_weight2) model_layers_14_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[89] alloc695: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc696: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc694, alloc691, model_layers_14_input_layernorm_weight2, alloc695, alloc696) R.vm.kill_object(alloc691) R.vm.kill_object(alloc694) R.vm.kill_object(model_layers_14_input_layernorm_weight2) lv270: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc695, alloc696 model_layers_14_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[85] alloc697: R.Tensor((1, 1, 2560), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2560]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul10(alloc695, model_layers_14_self_attn_qkv_proj_weight2, alloc697) R.vm.kill_object(alloc695) R.vm.kill_object(model_layers_14_self_attn_qkv_proj_weight2) lv96: R.Tensor((1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc697, R.shape([1, 10, 256]), sinfo_args=(R.Tensor((1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc697) alloc698: R.Tensor((1, 8, 256), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 8, 256]), R.dtype("float16")) _561: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(14), R.prim_value(T.float32(1.0)), lv96, alloc698) R.vm.kill_object(lv96) lv97: R.Tensor((1, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc698, R.shape([1, 1, 2048]), sinfo_args=(R.Tensor((1, 1, 2048), dtype="float16"),)) R.vm.kill_object(alloc698) model_layers_14_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[86] alloc699: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul11(lv97, model_layers_14_self_attn_o_proj_weight2, alloc699) R.vm.kill_object(lv97) R.vm.kill_object(model_layers_14_self_attn_o_proj_weight2) model_layers_14_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[90] alloc700: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc701: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc699, alloc696, model_layers_14_post_attention_layernorm_weight2, alloc700, alloc701) R.vm.kill_object(alloc696) R.vm.kill_object(alloc699) R.vm.kill_object(model_layers_14_post_attention_layernorm_weight2) lv272: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc700, alloc701 model_layers_14_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[87] alloc702: R.Tensor((1, 1, 32768), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 32768]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul12(alloc700, model_layers_14_mlp_gate_up_proj_weight2, alloc702) R.vm.kill_object(alloc700) R.vm.kill_object(model_layers_14_mlp_gate_up_proj_weight2) alloc703: R.Tensor((1, 1, 16384), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 16384]), R.dtype("float16")) cls.gemma_q0f16_fused_split2_gelu_tanh2_multiply5(alloc702, alloc703) R.vm.kill_object(alloc702) model_layers_14_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[88] alloc704: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul13(alloc703, model_layers_14_mlp_down_proj_weight2, alloc704) R.vm.kill_object(alloc703) R.vm.kill_object(model_layers_14_mlp_down_proj_weight2) model_layers_15_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[95] alloc705: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc706: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc704, alloc701, model_layers_15_input_layernorm_weight2, alloc705, alloc706) R.vm.kill_object(alloc701) R.vm.kill_object(alloc704) R.vm.kill_object(model_layers_15_input_layernorm_weight2) lv274: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc705, alloc706 model_layers_15_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[91] alloc707: R.Tensor((1, 1, 2560), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2560]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul10(alloc705, model_layers_15_self_attn_qkv_proj_weight2, alloc707) R.vm.kill_object(alloc705) R.vm.kill_object(model_layers_15_self_attn_qkv_proj_weight2) lv99: R.Tensor((1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc707, R.shape([1, 10, 256]), sinfo_args=(R.Tensor((1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc707) alloc708: R.Tensor((1, 8, 256), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 8, 256]), R.dtype("float16")) _569: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(15), R.prim_value(T.float32(1.0)), lv99, alloc708) R.vm.kill_object(lv99) lv100: R.Tensor((1, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc708, R.shape([1, 1, 2048]), sinfo_args=(R.Tensor((1, 1, 2048), dtype="float16"),)) R.vm.kill_object(alloc708) model_layers_15_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[92] alloc709: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul11(lv100, model_layers_15_self_attn_o_proj_weight2, alloc709) R.vm.kill_object(lv100) R.vm.kill_object(model_layers_15_self_attn_o_proj_weight2) model_layers_15_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[96] alloc710: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc711: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc709, alloc706, model_layers_15_post_attention_layernorm_weight2, alloc710, alloc711) R.vm.kill_object(alloc706) R.vm.kill_object(alloc709) R.vm.kill_object(model_layers_15_post_attention_layernorm_weight2) lv276: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc710, alloc711 model_layers_15_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[93] alloc712: R.Tensor((1, 1, 32768), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 32768]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul12(alloc710, model_layers_15_mlp_gate_up_proj_weight2, alloc712) R.vm.kill_object(alloc710) R.vm.kill_object(model_layers_15_mlp_gate_up_proj_weight2) alloc713: R.Tensor((1, 1, 16384), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 16384]), R.dtype("float16")) cls.gemma_q0f16_fused_split2_gelu_tanh2_multiply5(alloc712, alloc713) R.vm.kill_object(alloc712) model_layers_15_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[94] alloc714: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul13(alloc713, model_layers_15_mlp_down_proj_weight2, alloc714) R.vm.kill_object(alloc713) R.vm.kill_object(model_layers_15_mlp_down_proj_weight2) model_layers_16_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[101] alloc715: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc716: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc714, alloc711, model_layers_16_input_layernorm_weight2, alloc715, alloc716) R.vm.kill_object(alloc711) R.vm.kill_object(alloc714) R.vm.kill_object(model_layers_16_input_layernorm_weight2) lv278: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc715, alloc716 model_layers_16_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[97] alloc717: R.Tensor((1, 1, 2560), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2560]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul10(alloc715, model_layers_16_self_attn_qkv_proj_weight2, alloc717) R.vm.kill_object(alloc715) R.vm.kill_object(model_layers_16_self_attn_qkv_proj_weight2) lv102: R.Tensor((1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc717, R.shape([1, 10, 256]), sinfo_args=(R.Tensor((1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc717) alloc718: R.Tensor((1, 8, 256), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 8, 256]), R.dtype("float16")) _577: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(16), R.prim_value(T.float32(1.0)), lv102, alloc718) R.vm.kill_object(lv102) lv103: R.Tensor((1, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc718, R.shape([1, 1, 2048]), sinfo_args=(R.Tensor((1, 1, 2048), dtype="float16"),)) R.vm.kill_object(alloc718) model_layers_16_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[98] alloc719: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul11(lv103, model_layers_16_self_attn_o_proj_weight2, alloc719) R.vm.kill_object(lv103) R.vm.kill_object(model_layers_16_self_attn_o_proj_weight2) model_layers_16_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[102] alloc720: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc721: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc719, alloc716, model_layers_16_post_attention_layernorm_weight2, alloc720, alloc721) R.vm.kill_object(alloc716) R.vm.kill_object(alloc719) R.vm.kill_object(model_layers_16_post_attention_layernorm_weight2) lv280: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc720, alloc721 model_layers_16_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[99] alloc722: R.Tensor((1, 1, 32768), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 32768]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul12(alloc720, model_layers_16_mlp_gate_up_proj_weight2, alloc722) R.vm.kill_object(alloc720) R.vm.kill_object(model_layers_16_mlp_gate_up_proj_weight2) alloc723: R.Tensor((1, 1, 16384), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 16384]), R.dtype("float16")) cls.gemma_q0f16_fused_split2_gelu_tanh2_multiply5(alloc722, alloc723) R.vm.kill_object(alloc722) model_layers_16_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[100] alloc724: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul13(alloc723, model_layers_16_mlp_down_proj_weight2, alloc724) R.vm.kill_object(alloc723) R.vm.kill_object(model_layers_16_mlp_down_proj_weight2) model_layers_17_input_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[107] alloc725: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc726: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc724, alloc721, model_layers_17_input_layernorm_weight2, alloc725, alloc726) R.vm.kill_object(alloc721) R.vm.kill_object(alloc724) R.vm.kill_object(model_layers_17_input_layernorm_weight2) lv282: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc725, alloc726 model_layers_17_self_attn_qkv_proj_weight2: R.Tensor((2560, 2048), dtype="float16") = packed_params[103] alloc727: R.Tensor((1, 1, 2560), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2560]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul10(alloc725, model_layers_17_self_attn_qkv_proj_weight2, alloc727) R.vm.kill_object(alloc725) R.vm.kill_object(model_layers_17_self_attn_qkv_proj_weight2) lv105: R.Tensor((1, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc727, R.shape([1, 10, 256]), sinfo_args=(R.Tensor((1, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc727) alloc728: R.Tensor((1, 8, 256), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 8, 256]), R.dtype("float16")) _585: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(17), R.prim_value(T.float32(1.0)), lv105, alloc728) R.vm.kill_object(lv105) lv106: R.Tensor((1, 1, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc728, R.shape([1, 1, 2048]), sinfo_args=(R.Tensor((1, 1, 2048), dtype="float16"),)) R.vm.kill_object(alloc728) model_layers_17_self_attn_o_proj_weight2: R.Tensor((2048, 2048), dtype="float16") = packed_params[104] alloc729: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul11(lv106, model_layers_17_self_attn_o_proj_weight2, alloc729) R.vm.kill_object(lv106) R.vm.kill_object(model_layers_17_self_attn_o_proj_weight2) model_layers_17_post_attention_layernorm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[108] alloc730: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) alloc731: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage22, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) R.vm.kill_object(storage22) cls.gemma_q0f16_fuse_add_norm_prefill(alloc729, alloc726, model_layers_17_post_attention_layernorm_weight2, alloc730, alloc731) R.vm.kill_object(alloc726) R.vm.kill_object(alloc729) R.vm.kill_object(model_layers_17_post_attention_layernorm_weight2) lv284: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc730, alloc731 model_layers_17_mlp_gate_up_proj_weight2: R.Tensor((32768, 2048), dtype="float16") = packed_params[105] alloc732: R.Tensor((1, 1, 32768), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 32768]), R.dtype("float16")) cls.gemma_q0f16_NT_matmul12(alloc730, model_layers_17_mlp_gate_up_proj_weight2, alloc732) R.vm.kill_object(alloc730) R.vm.kill_object(model_layers_17_mlp_gate_up_proj_weight2) alloc733: R.Tensor((1, 1, 16384), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 16384]), R.dtype("float16")) cls.gemma_q0f16_fused_split2_gelu_tanh2_multiply5(alloc732, alloc733) R.vm.kill_object(alloc732) model_layers_17_mlp_down_proj_weight2: R.Tensor((2048, 16384), dtype="float16") = packed_params[106] alloc734: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage21, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) R.vm.kill_object(storage21) cls.gemma_q0f16_NT_matmul13(alloc733, model_layers_17_mlp_down_proj_weight2, alloc734) R.vm.kill_object(alloc733) R.vm.kill_object(model_layers_17_mlp_down_proj_weight2) model_norm_weight2: R.Tensor((2048,), dtype="float16") = packed_params[109] alloc735: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage24, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) R.vm.kill_object(storage24) alloc736: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage23, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) R.vm.kill_object(storage23) cls.gemma_q0f16_fuse_add_norm_prefill(alloc734, alloc731, model_norm_weight2, alloc735, alloc736) R.vm.kill_object(alloc731) R.vm.kill_object(alloc734) R.vm.kill_object(model_norm_weight2) lv286: R.Tuple(R.Tensor((1, 1, 2048), dtype="float16"), R.Tensor((1, 1, 2048), dtype="float16")) = alloc735, alloc736 R.vm.kill_object(alloc736) model_embed_tokens_weight2: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[0] gv786: R.Shape(ndim=1) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(2), sinfo_args=(R.Shape(ndim=1),)) storage25: R.Object = R.vm.alloc_storage(gv786, R.prim_value(0), R.dtype("uint8"), R.str("global")) gv787: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), sinfo_args=(R.Shape(ndim=3),)) alloc737: R.Tensor(dtype="float32", ndim=3) = R.vm.alloc_tensor(storage25, R.prim_value(0), gv787, R.dtype("float32")) R.vm.kill_object(storage25) cls.gemma_q0f16_NT_matmul14(alloc735, model_embed_tokens_weight2, alloc737) R.vm.kill_object(alloc735) R.vm.kill_object(model_embed_tokens_weight2) gv2: R.Tuple(R.Tensor(dtype="float32", ndim=3), R.Object) = alloc737, paged_kv_cache R.vm.kill_object(alloc737) gv788: R.Tensor(dtype="float32", ndim=3) = gv2[0] R.call_packed("vm.builtin.match_shape", gv788, shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(3), R.prim_value(0), R.str("ErrorContext(fn=decode, loc=return, annotation=R.Tuple(R.Tensor((1, 1, vocab_size), dtype=\"float32\"), R.Object)) "), sinfo_args=(R.Tuple,)) return gv2 @R.function def embed(input_ids: R.Tensor(("seq_len",), dtype="int32"), packed_params: R.Tuple(R.Tensor(("vocab_size", 2048), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"))) -> R.Tensor(("seq_len", 2048), dtype="float16"): seq_len = T.int64() vocab_size = T.int64() R.func_attr({"num_input": 1, "relax.force_pure": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 8192, "total_seq_len": 8192}}) cls = Module shape_heap: R.Tensor(dtype="int64", ndim=1) = R.call_builtin_with_ctx("vm.builtin.alloc_shape_heap", (R.prim_value(2),), sinfo_args=(R.Tensor(dtype="int64", ndim=1),)) R.call_packed("vm.builtin.check_tensor_info", input_ids, R.prim_value(1), R.dtype("int32"), R.str("ErrorContext(fn=embed, loc=param[0], param=input_ids, annotation=R.Tensor((seq_len,), dtype=\"int32\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_tuple_info", packed_params, R.prim_value(110), R.str("ErrorContext(fn=embed, loc=param[1], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 2048), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv524: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[0] R.call_packed("vm.builtin.check_tensor_info", gv524, R.prim_value(2), R.dtype("float16"), R.str("ErrorContext(fn=embed, loc=param[1], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 2048), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", input_ids, shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(0), R.str("ErrorContext(fn=embed, loc=param[0], param=input_ids, annotation=R.Tensor((seq_len,), dtype=\"int32\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv524, shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(2048), R.str("ErrorContext(fn=embed, loc=param[1], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 2048), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) model_embed_tokens_weight: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[0] storage14: R.Object = R.vm.alloc_storage(R.shape([33554432]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv525: R.Shape(ndim=2) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=2),)) alloc370: R.Tensor(dtype="float16", ndim=2) = R.vm.alloc_tensor(storage14, R.prim_value(0), gv525, R.dtype("float16")) R.vm.kill_object(storage14) cls.gemma_q0f16_take1(model_embed_tokens_weight, input_ids, alloc370) R.vm.kill_object(model_embed_tokens_weight) R.call_packed("vm.builtin.match_shape", alloc370, shape_heap, R.prim_value(2), R.prim_value(3), R.prim_value(0), R.prim_value(0), R.prim_value(2048), R.str("ErrorContext(fn=embed, loc=return, annotation=R.Tensor((seq_len, 2048), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) return alloc370 @R.function def prefill(input_embed: R.Tensor((1, "seq_len", 2048), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 2048), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2560, 2048), dtype="float16"), R.Tensor((2048, 2048), dtype="float16"), R.Tensor((32768, 2048), dtype="float16"), R.Tensor((2048, 16384), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"), R.Tensor((2048,), dtype="float16"))) -> R.Tuple(R.Tensor((1, 1, "vocab_size"), dtype="float32"), R.Object): vocab_size = T.int64() seq_len = T.int64() R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.force_pure": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 8192, "total_seq_len": 8192}}) cls = Module shape_heap: R.Tensor(dtype="int64", ndim=1) = R.call_builtin_with_ctx("vm.builtin.alloc_shape_heap", (R.prim_value(4),), sinfo_args=(R.Tensor(dtype="int64", ndim=1),)) R.call_packed("vm.builtin.check_tensor_info", input_embed, R.prim_value(3), R.dtype("float16"), R.str("ErrorContext(fn=prefill, loc=param[0], param=input_embed, annotation=R.Tensor((1, seq_len, 2048), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_tuple_info", packed_params, R.prim_value(110), R.str("ErrorContext(fn=prefill, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 2048), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) gv266: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[0] R.call_packed("vm.builtin.check_tensor_info", gv266, R.prim_value(2), R.dtype("float16"), R.str("ErrorContext(fn=prefill, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 2048), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", input_embed, shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), R.str("ErrorContext(fn=prefill, loc=param[0], param=input_embed, annotation=R.Tensor((1, seq_len, 2048), dtype=\"float16\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", gv266, shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(2048), R.str("ErrorContext(fn=prefill, loc=param[2], param=packed_params, annotation=R.Tuple(R.Tensor((vocab_size, 2048), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2560, 2048), dtype=\"float16\"), R.Tensor((2048, 2048), dtype=\"float16\"), R.Tensor((32768, 2048), dtype=\"float16\"), R.Tensor((2048, 16384), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"), R.Tensor((2048,), dtype=\"float16\"))) "), sinfo_args=(R.Tuple,)) cls.gemma_q0f16_shape_func2(shape_heap) storage8: R.Object = R.vm.alloc_storage(R.shape([33554432]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv267: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc186: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv267, R.dtype("float16")) cls.gemma_q0f16_multiply2(input_embed, alloc186) model_layers_0_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[5] storage9: R.Object = R.vm.alloc_storage(R.shape([268435456]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv268: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc187: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv268, R.dtype("float16")) cls.gemma_q0f16_rms_norm1(alloc186, model_layers_0_input_layernorm_weight1, alloc187) R.vm.kill_object(model_layers_0_input_layernorm_weight1) model_layers_0_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[1] storage10: R.Object = R.vm.alloc_storage(R.shape([536870912]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv269: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc188: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv269, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc187, model_layers_0_self_attn_qkv_proj_weight1, alloc188) R.vm.kill_object(alloc187) R.vm.kill_object(model_layers_0_self_attn_qkv_proj_weight1) gv270: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc188, gv270, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc188) gv271: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape1: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape, gv271, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape) gv272: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc189: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv272, R.dtype("float16")) _152: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape1, alloc189) R.vm.kill_object(reshape1) gv273: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape2: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc189, gv273, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc189) gv274: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape3: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape2, gv274, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape2) model_layers_0_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[2] gv275: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc190: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv275, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape3, model_layers_0_self_attn_o_proj_weight1, alloc190) R.vm.kill_object(reshape3) R.vm.kill_object(model_layers_0_self_attn_o_proj_weight1) model_layers_0_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[6] gv276: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc191: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv276, R.dtype("float16")) storage11: R.Object = R.vm.alloc_storage(R.shape([268435456]), R.prim_value(0), R.dtype("uint8"), R.str("global")) gv277: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc192: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv277, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc190, alloc186, model_layers_0_post_attention_layernorm_weight1, alloc191, alloc192) R.vm.kill_object(alloc186) R.vm.kill_object(alloc190) R.vm.kill_object(model_layers_0_post_attention_layernorm_weight1) lv288: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc191, alloc192 model_layers_0_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[3] gv278: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc193: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv278, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc191, model_layers_0_mlp_gate_up_proj_weight1, alloc193) R.vm.kill_object(alloc191) R.vm.kill_object(model_layers_0_mlp_gate_up_proj_weight1) gv279: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc194: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv279, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc193, alloc194) R.vm.kill_object(alloc193) model_layers_0_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[4] gv280: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc195: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv280, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc194, model_layers_0_mlp_down_proj_weight1, alloc195) R.vm.kill_object(alloc194) R.vm.kill_object(model_layers_0_mlp_down_proj_weight1) model_layers_1_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[11] gv281: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc196: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv281, R.dtype("float16")) gv282: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc197: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv282, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc195, alloc192, model_layers_1_input_layernorm_weight1, alloc196, alloc197) R.vm.kill_object(alloc192) R.vm.kill_object(alloc195) R.vm.kill_object(model_layers_1_input_layernorm_weight1) lv290: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc196, alloc197 model_layers_1_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[7] gv283: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc198: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv283, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc196, model_layers_1_self_attn_qkv_proj_weight1, alloc198) R.vm.kill_object(alloc196) R.vm.kill_object(model_layers_1_self_attn_qkv_proj_weight1) gv284: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape4: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc198, gv284, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc198) gv285: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape5: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape4, gv285, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape4) gv286: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc199: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv286, R.dtype("float16")) _160: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape5, alloc199) R.vm.kill_object(reshape5) gv287: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape6: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc199, gv287, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc199) gv288: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape7: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape6, gv288, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape6) model_layers_1_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[8] gv289: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc200: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv289, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape7, model_layers_1_self_attn_o_proj_weight1, alloc200) R.vm.kill_object(reshape7) R.vm.kill_object(model_layers_1_self_attn_o_proj_weight1) model_layers_1_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[12] gv290: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc201: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv290, R.dtype("float16")) gv291: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc202: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv291, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc200, alloc197, model_layers_1_post_attention_layernorm_weight1, alloc201, alloc202) R.vm.kill_object(alloc197) R.vm.kill_object(alloc200) R.vm.kill_object(model_layers_1_post_attention_layernorm_weight1) lv292: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc201, alloc202 model_layers_1_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[9] gv292: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc203: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv292, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc201, model_layers_1_mlp_gate_up_proj_weight1, alloc203) R.vm.kill_object(alloc201) R.vm.kill_object(model_layers_1_mlp_gate_up_proj_weight1) gv293: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc204: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv293, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc203, alloc204) R.vm.kill_object(alloc203) model_layers_1_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[10] gv294: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc205: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv294, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc204, model_layers_1_mlp_down_proj_weight1, alloc205) R.vm.kill_object(alloc204) R.vm.kill_object(model_layers_1_mlp_down_proj_weight1) model_layers_2_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[17] gv295: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc206: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv295, R.dtype("float16")) gv296: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc207: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv296, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc205, alloc202, model_layers_2_input_layernorm_weight1, alloc206, alloc207) R.vm.kill_object(alloc202) R.vm.kill_object(alloc205) R.vm.kill_object(model_layers_2_input_layernorm_weight1) lv294: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc206, alloc207 model_layers_2_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[13] gv297: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc208: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv297, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc206, model_layers_2_self_attn_qkv_proj_weight1, alloc208) R.vm.kill_object(alloc206) R.vm.kill_object(model_layers_2_self_attn_qkv_proj_weight1) gv298: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape8: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc208, gv298, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc208) gv299: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape9: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape8, gv299, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape8) gv300: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc209: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv300, R.dtype("float16")) _168: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape9, alloc209) R.vm.kill_object(reshape9) gv301: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape10: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc209, gv301, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc209) gv302: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape11: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape10, gv302, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape10) model_layers_2_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[14] gv303: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc210: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv303, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape11, model_layers_2_self_attn_o_proj_weight1, alloc210) R.vm.kill_object(reshape11) R.vm.kill_object(model_layers_2_self_attn_o_proj_weight1) model_layers_2_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[18] gv304: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc211: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv304, R.dtype("float16")) gv305: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc212: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv305, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc210, alloc207, model_layers_2_post_attention_layernorm_weight1, alloc211, alloc212) R.vm.kill_object(alloc207) R.vm.kill_object(alloc210) R.vm.kill_object(model_layers_2_post_attention_layernorm_weight1) lv296: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc211, alloc212 model_layers_2_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[15] gv306: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc213: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv306, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc211, model_layers_2_mlp_gate_up_proj_weight1, alloc213) R.vm.kill_object(alloc211) R.vm.kill_object(model_layers_2_mlp_gate_up_proj_weight1) gv307: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc214: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv307, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc213, alloc214) R.vm.kill_object(alloc213) model_layers_2_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[16] gv308: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc215: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv308, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc214, model_layers_2_mlp_down_proj_weight1, alloc215) R.vm.kill_object(alloc214) R.vm.kill_object(model_layers_2_mlp_down_proj_weight1) model_layers_3_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[23] gv309: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc216: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv309, R.dtype("float16")) gv310: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc217: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv310, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc215, alloc212, model_layers_3_input_layernorm_weight1, alloc216, alloc217) R.vm.kill_object(alloc212) R.vm.kill_object(alloc215) R.vm.kill_object(model_layers_3_input_layernorm_weight1) lv298: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc216, alloc217 model_layers_3_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[19] gv311: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc218: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv311, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc216, model_layers_3_self_attn_qkv_proj_weight1, alloc218) R.vm.kill_object(alloc216) R.vm.kill_object(model_layers_3_self_attn_qkv_proj_weight1) gv312: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape12: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc218, gv312, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc218) gv313: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape13: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape12, gv313, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape12) gv314: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc219: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv314, R.dtype("float16")) _176: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape13, alloc219) R.vm.kill_object(reshape13) gv315: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape14: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc219, gv315, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc219) gv316: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape15: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape14, gv316, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape14) model_layers_3_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[20] gv317: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc220: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv317, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape15, model_layers_3_self_attn_o_proj_weight1, alloc220) R.vm.kill_object(reshape15) R.vm.kill_object(model_layers_3_self_attn_o_proj_weight1) model_layers_3_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[24] gv318: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc221: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv318, R.dtype("float16")) gv319: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc222: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv319, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc220, alloc217, model_layers_3_post_attention_layernorm_weight1, alloc221, alloc222) R.vm.kill_object(alloc217) R.vm.kill_object(alloc220) R.vm.kill_object(model_layers_3_post_attention_layernorm_weight1) lv300: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc221, alloc222 model_layers_3_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[21] gv320: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc223: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv320, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc221, model_layers_3_mlp_gate_up_proj_weight1, alloc223) R.vm.kill_object(alloc221) R.vm.kill_object(model_layers_3_mlp_gate_up_proj_weight1) gv321: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc224: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv321, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc223, alloc224) R.vm.kill_object(alloc223) model_layers_3_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[22] gv322: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc225: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv322, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc224, model_layers_3_mlp_down_proj_weight1, alloc225) R.vm.kill_object(alloc224) R.vm.kill_object(model_layers_3_mlp_down_proj_weight1) model_layers_4_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[29] gv323: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc226: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv323, R.dtype("float16")) gv324: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc227: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv324, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc225, alloc222, model_layers_4_input_layernorm_weight1, alloc226, alloc227) R.vm.kill_object(alloc222) R.vm.kill_object(alloc225) R.vm.kill_object(model_layers_4_input_layernorm_weight1) lv302: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc226, alloc227 model_layers_4_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[25] gv325: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc228: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv325, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc226, model_layers_4_self_attn_qkv_proj_weight1, alloc228) R.vm.kill_object(alloc226) R.vm.kill_object(model_layers_4_self_attn_qkv_proj_weight1) gv326: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape16: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc228, gv326, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc228) gv327: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape17: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape16, gv327, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape16) gv328: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc229: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv328, R.dtype("float16")) _184: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape17, alloc229) R.vm.kill_object(reshape17) gv329: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape18: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc229, gv329, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc229) gv330: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape19: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape18, gv330, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape18) model_layers_4_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[26] gv331: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc230: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv331, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape19, model_layers_4_self_attn_o_proj_weight1, alloc230) R.vm.kill_object(reshape19) R.vm.kill_object(model_layers_4_self_attn_o_proj_weight1) model_layers_4_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[30] gv332: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc231: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv332, R.dtype("float16")) gv333: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc232: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv333, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc230, alloc227, model_layers_4_post_attention_layernorm_weight1, alloc231, alloc232) R.vm.kill_object(alloc227) R.vm.kill_object(alloc230) R.vm.kill_object(model_layers_4_post_attention_layernorm_weight1) lv304: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc231, alloc232 model_layers_4_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[27] gv334: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc233: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv334, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc231, model_layers_4_mlp_gate_up_proj_weight1, alloc233) R.vm.kill_object(alloc231) R.vm.kill_object(model_layers_4_mlp_gate_up_proj_weight1) gv335: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc234: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv335, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc233, alloc234) R.vm.kill_object(alloc233) model_layers_4_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[28] gv336: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc235: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv336, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc234, model_layers_4_mlp_down_proj_weight1, alloc235) R.vm.kill_object(alloc234) R.vm.kill_object(model_layers_4_mlp_down_proj_weight1) model_layers_5_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[35] gv337: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc236: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv337, R.dtype("float16")) gv338: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc237: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv338, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc235, alloc232, model_layers_5_input_layernorm_weight1, alloc236, alloc237) R.vm.kill_object(alloc232) R.vm.kill_object(alloc235) R.vm.kill_object(model_layers_5_input_layernorm_weight1) lv306: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc236, alloc237 model_layers_5_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[31] gv339: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc238: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv339, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc236, model_layers_5_self_attn_qkv_proj_weight1, alloc238) R.vm.kill_object(alloc236) R.vm.kill_object(model_layers_5_self_attn_qkv_proj_weight1) gv340: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape20: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc238, gv340, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc238) gv341: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape21: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape20, gv341, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape20) gv342: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc239: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv342, R.dtype("float16")) _192: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape21, alloc239) R.vm.kill_object(reshape21) gv343: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape22: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc239, gv343, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc239) gv344: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape23: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape22, gv344, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape22) model_layers_5_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[32] gv345: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc240: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv345, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape23, model_layers_5_self_attn_o_proj_weight1, alloc240) R.vm.kill_object(reshape23) R.vm.kill_object(model_layers_5_self_attn_o_proj_weight1) model_layers_5_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[36] gv346: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc241: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv346, R.dtype("float16")) gv347: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc242: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv347, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc240, alloc237, model_layers_5_post_attention_layernorm_weight1, alloc241, alloc242) R.vm.kill_object(alloc237) R.vm.kill_object(alloc240) R.vm.kill_object(model_layers_5_post_attention_layernorm_weight1) lv308: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc241, alloc242 model_layers_5_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[33] gv348: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc243: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv348, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc241, model_layers_5_mlp_gate_up_proj_weight1, alloc243) R.vm.kill_object(alloc241) R.vm.kill_object(model_layers_5_mlp_gate_up_proj_weight1) gv349: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc244: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv349, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc243, alloc244) R.vm.kill_object(alloc243) model_layers_5_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[34] gv350: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc245: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv350, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc244, model_layers_5_mlp_down_proj_weight1, alloc245) R.vm.kill_object(alloc244) R.vm.kill_object(model_layers_5_mlp_down_proj_weight1) model_layers_6_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[41] gv351: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc246: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv351, R.dtype("float16")) gv352: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc247: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv352, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc245, alloc242, model_layers_6_input_layernorm_weight1, alloc246, alloc247) R.vm.kill_object(alloc242) R.vm.kill_object(alloc245) R.vm.kill_object(model_layers_6_input_layernorm_weight1) lv310: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc246, alloc247 model_layers_6_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[37] gv353: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc248: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv353, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc246, model_layers_6_self_attn_qkv_proj_weight1, alloc248) R.vm.kill_object(alloc246) R.vm.kill_object(model_layers_6_self_attn_qkv_proj_weight1) gv354: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape24: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc248, gv354, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc248) gv355: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape25: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape24, gv355, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape24) gv356: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc249: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv356, R.dtype("float16")) _200: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(6), R.prim_value(T.float32(1.0)), reshape25, alloc249) R.vm.kill_object(reshape25) gv357: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape26: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc249, gv357, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc249) gv358: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape27: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape26, gv358, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape26) model_layers_6_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[38] gv359: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc250: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv359, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape27, model_layers_6_self_attn_o_proj_weight1, alloc250) R.vm.kill_object(reshape27) R.vm.kill_object(model_layers_6_self_attn_o_proj_weight1) model_layers_6_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[42] gv360: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc251: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv360, R.dtype("float16")) gv361: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc252: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv361, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc250, alloc247, model_layers_6_post_attention_layernorm_weight1, alloc251, alloc252) R.vm.kill_object(alloc247) R.vm.kill_object(alloc250) R.vm.kill_object(model_layers_6_post_attention_layernorm_weight1) lv312: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc251, alloc252 model_layers_6_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[39] gv362: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc253: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv362, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc251, model_layers_6_mlp_gate_up_proj_weight1, alloc253) R.vm.kill_object(alloc251) R.vm.kill_object(model_layers_6_mlp_gate_up_proj_weight1) gv363: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc254: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv363, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc253, alloc254) R.vm.kill_object(alloc253) model_layers_6_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[40] gv364: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc255: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv364, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc254, model_layers_6_mlp_down_proj_weight1, alloc255) R.vm.kill_object(alloc254) R.vm.kill_object(model_layers_6_mlp_down_proj_weight1) model_layers_7_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[47] gv365: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc256: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv365, R.dtype("float16")) gv366: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc257: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv366, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc255, alloc252, model_layers_7_input_layernorm_weight1, alloc256, alloc257) R.vm.kill_object(alloc252) R.vm.kill_object(alloc255) R.vm.kill_object(model_layers_7_input_layernorm_weight1) lv314: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc256, alloc257 model_layers_7_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[43] gv367: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc258: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv367, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc256, model_layers_7_self_attn_qkv_proj_weight1, alloc258) R.vm.kill_object(alloc256) R.vm.kill_object(model_layers_7_self_attn_qkv_proj_weight1) gv368: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape28: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc258, gv368, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc258) gv369: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape29: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape28, gv369, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape28) gv370: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc259: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv370, R.dtype("float16")) _208: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(7), R.prim_value(T.float32(1.0)), reshape29, alloc259) R.vm.kill_object(reshape29) gv371: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape30: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc259, gv371, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc259) gv372: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape31: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape30, gv372, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape30) model_layers_7_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[44] gv373: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc260: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv373, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape31, model_layers_7_self_attn_o_proj_weight1, alloc260) R.vm.kill_object(reshape31) R.vm.kill_object(model_layers_7_self_attn_o_proj_weight1) model_layers_7_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[48] gv374: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc261: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv374, R.dtype("float16")) gv375: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc262: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv375, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc260, alloc257, model_layers_7_post_attention_layernorm_weight1, alloc261, alloc262) R.vm.kill_object(alloc257) R.vm.kill_object(alloc260) R.vm.kill_object(model_layers_7_post_attention_layernorm_weight1) lv316: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc261, alloc262 model_layers_7_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[45] gv376: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc263: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv376, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc261, model_layers_7_mlp_gate_up_proj_weight1, alloc263) R.vm.kill_object(alloc261) R.vm.kill_object(model_layers_7_mlp_gate_up_proj_weight1) gv377: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc264: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv377, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc263, alloc264) R.vm.kill_object(alloc263) model_layers_7_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[46] gv378: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc265: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv378, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc264, model_layers_7_mlp_down_proj_weight1, alloc265) R.vm.kill_object(alloc264) R.vm.kill_object(model_layers_7_mlp_down_proj_weight1) model_layers_8_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[53] gv379: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc266: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv379, R.dtype("float16")) gv380: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc267: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv380, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc265, alloc262, model_layers_8_input_layernorm_weight1, alloc266, alloc267) R.vm.kill_object(alloc262) R.vm.kill_object(alloc265) R.vm.kill_object(model_layers_8_input_layernorm_weight1) lv318: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc266, alloc267 model_layers_8_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[49] gv381: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc268: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv381, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc266, model_layers_8_self_attn_qkv_proj_weight1, alloc268) R.vm.kill_object(alloc266) R.vm.kill_object(model_layers_8_self_attn_qkv_proj_weight1) gv382: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape32: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc268, gv382, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc268) gv383: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape33: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape32, gv383, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape32) gv384: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc269: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv384, R.dtype("float16")) _216: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(8), R.prim_value(T.float32(1.0)), reshape33, alloc269) R.vm.kill_object(reshape33) gv385: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape34: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc269, gv385, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc269) gv386: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape35: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape34, gv386, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape34) model_layers_8_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[50] gv387: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc270: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv387, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape35, model_layers_8_self_attn_o_proj_weight1, alloc270) R.vm.kill_object(reshape35) R.vm.kill_object(model_layers_8_self_attn_o_proj_weight1) model_layers_8_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[54] gv388: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc271: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv388, R.dtype("float16")) gv389: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc272: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv389, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc270, alloc267, model_layers_8_post_attention_layernorm_weight1, alloc271, alloc272) R.vm.kill_object(alloc267) R.vm.kill_object(alloc270) R.vm.kill_object(model_layers_8_post_attention_layernorm_weight1) lv320: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc271, alloc272 model_layers_8_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[51] gv390: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc273: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv390, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc271, model_layers_8_mlp_gate_up_proj_weight1, alloc273) R.vm.kill_object(alloc271) R.vm.kill_object(model_layers_8_mlp_gate_up_proj_weight1) gv391: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc274: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv391, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc273, alloc274) R.vm.kill_object(alloc273) model_layers_8_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[52] gv392: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc275: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv392, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc274, model_layers_8_mlp_down_proj_weight1, alloc275) R.vm.kill_object(alloc274) R.vm.kill_object(model_layers_8_mlp_down_proj_weight1) model_layers_9_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[59] gv393: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc276: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv393, R.dtype("float16")) gv394: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc277: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv394, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc275, alloc272, model_layers_9_input_layernorm_weight1, alloc276, alloc277) R.vm.kill_object(alloc272) R.vm.kill_object(alloc275) R.vm.kill_object(model_layers_9_input_layernorm_weight1) lv322: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc276, alloc277 model_layers_9_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[55] gv395: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc278: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv395, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc276, model_layers_9_self_attn_qkv_proj_weight1, alloc278) R.vm.kill_object(alloc276) R.vm.kill_object(model_layers_9_self_attn_qkv_proj_weight1) gv396: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape36: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc278, gv396, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc278) gv397: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape37: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape36, gv397, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape36) gv398: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc279: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv398, R.dtype("float16")) _224: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(9), R.prim_value(T.float32(1.0)), reshape37, alloc279) R.vm.kill_object(reshape37) gv399: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape38: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc279, gv399, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc279) gv400: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape39: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape38, gv400, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape38) model_layers_9_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[56] gv401: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc280: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv401, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape39, model_layers_9_self_attn_o_proj_weight1, alloc280) R.vm.kill_object(reshape39) R.vm.kill_object(model_layers_9_self_attn_o_proj_weight1) model_layers_9_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[60] gv402: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc281: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv402, R.dtype("float16")) gv403: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc282: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv403, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc280, alloc277, model_layers_9_post_attention_layernorm_weight1, alloc281, alloc282) R.vm.kill_object(alloc277) R.vm.kill_object(alloc280) R.vm.kill_object(model_layers_9_post_attention_layernorm_weight1) lv324: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc281, alloc282 model_layers_9_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[57] gv404: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc283: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv404, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc281, model_layers_9_mlp_gate_up_proj_weight1, alloc283) R.vm.kill_object(alloc281) R.vm.kill_object(model_layers_9_mlp_gate_up_proj_weight1) gv405: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc284: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv405, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc283, alloc284) R.vm.kill_object(alloc283) model_layers_9_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[58] gv406: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc285: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv406, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc284, model_layers_9_mlp_down_proj_weight1, alloc285) R.vm.kill_object(alloc284) R.vm.kill_object(model_layers_9_mlp_down_proj_weight1) model_layers_10_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[65] gv407: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc286: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv407, R.dtype("float16")) gv408: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc287: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv408, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc285, alloc282, model_layers_10_input_layernorm_weight1, alloc286, alloc287) R.vm.kill_object(alloc282) R.vm.kill_object(alloc285) R.vm.kill_object(model_layers_10_input_layernorm_weight1) lv326: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc286, alloc287 model_layers_10_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[61] gv409: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc288: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv409, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc286, model_layers_10_self_attn_qkv_proj_weight1, alloc288) R.vm.kill_object(alloc286) R.vm.kill_object(model_layers_10_self_attn_qkv_proj_weight1) gv410: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape40: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc288, gv410, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc288) gv411: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape41: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape40, gv411, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape40) gv412: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc289: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv412, R.dtype("float16")) _232: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(10), R.prim_value(T.float32(1.0)), reshape41, alloc289) R.vm.kill_object(reshape41) gv413: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape42: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc289, gv413, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc289) gv414: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape43: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape42, gv414, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape42) model_layers_10_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[62] gv415: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc290: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv415, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape43, model_layers_10_self_attn_o_proj_weight1, alloc290) R.vm.kill_object(reshape43) R.vm.kill_object(model_layers_10_self_attn_o_proj_weight1) model_layers_10_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[66] gv416: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc291: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv416, R.dtype("float16")) gv417: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc292: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv417, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc290, alloc287, model_layers_10_post_attention_layernorm_weight1, alloc291, alloc292) R.vm.kill_object(alloc287) R.vm.kill_object(alloc290) R.vm.kill_object(model_layers_10_post_attention_layernorm_weight1) lv328: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc291, alloc292 model_layers_10_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[63] gv418: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc293: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv418, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc291, model_layers_10_mlp_gate_up_proj_weight1, alloc293) R.vm.kill_object(alloc291) R.vm.kill_object(model_layers_10_mlp_gate_up_proj_weight1) gv419: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc294: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv419, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc293, alloc294) R.vm.kill_object(alloc293) model_layers_10_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[64] gv420: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc295: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv420, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc294, model_layers_10_mlp_down_proj_weight1, alloc295) R.vm.kill_object(alloc294) R.vm.kill_object(model_layers_10_mlp_down_proj_weight1) model_layers_11_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[71] gv421: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc296: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv421, R.dtype("float16")) gv422: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc297: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv422, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc295, alloc292, model_layers_11_input_layernorm_weight1, alloc296, alloc297) R.vm.kill_object(alloc292) R.vm.kill_object(alloc295) R.vm.kill_object(model_layers_11_input_layernorm_weight1) lv330: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc296, alloc297 model_layers_11_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[67] gv423: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc298: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv423, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc296, model_layers_11_self_attn_qkv_proj_weight1, alloc298) R.vm.kill_object(alloc296) R.vm.kill_object(model_layers_11_self_attn_qkv_proj_weight1) gv424: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape44: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc298, gv424, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc298) gv425: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape45: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape44, gv425, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape44) gv426: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc299: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv426, R.dtype("float16")) _240: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(11), R.prim_value(T.float32(1.0)), reshape45, alloc299) R.vm.kill_object(reshape45) gv427: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape46: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc299, gv427, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc299) gv428: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape47: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape46, gv428, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape46) model_layers_11_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[68] gv429: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc300: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv429, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape47, model_layers_11_self_attn_o_proj_weight1, alloc300) R.vm.kill_object(reshape47) R.vm.kill_object(model_layers_11_self_attn_o_proj_weight1) model_layers_11_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[72] gv430: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc301: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv430, R.dtype("float16")) gv431: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc302: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv431, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc300, alloc297, model_layers_11_post_attention_layernorm_weight1, alloc301, alloc302) R.vm.kill_object(alloc297) R.vm.kill_object(alloc300) R.vm.kill_object(model_layers_11_post_attention_layernorm_weight1) lv332: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc301, alloc302 model_layers_11_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[69] gv432: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc303: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv432, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc301, model_layers_11_mlp_gate_up_proj_weight1, alloc303) R.vm.kill_object(alloc301) R.vm.kill_object(model_layers_11_mlp_gate_up_proj_weight1) gv433: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc304: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv433, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc303, alloc304) R.vm.kill_object(alloc303) model_layers_11_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[70] gv434: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc305: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv434, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc304, model_layers_11_mlp_down_proj_weight1, alloc305) R.vm.kill_object(alloc304) R.vm.kill_object(model_layers_11_mlp_down_proj_weight1) model_layers_12_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[77] gv435: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc306: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv435, R.dtype("float16")) gv436: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc307: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv436, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc305, alloc302, model_layers_12_input_layernorm_weight1, alloc306, alloc307) R.vm.kill_object(alloc302) R.vm.kill_object(alloc305) R.vm.kill_object(model_layers_12_input_layernorm_weight1) lv334: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc306, alloc307 model_layers_12_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[73] gv437: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc308: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv437, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc306, model_layers_12_self_attn_qkv_proj_weight1, alloc308) R.vm.kill_object(alloc306) R.vm.kill_object(model_layers_12_self_attn_qkv_proj_weight1) gv438: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape48: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc308, gv438, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc308) gv439: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape49: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape48, gv439, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape48) gv440: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc309: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv440, R.dtype("float16")) _248: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(12), R.prim_value(T.float32(1.0)), reshape49, alloc309) R.vm.kill_object(reshape49) gv441: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape50: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc309, gv441, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc309) gv442: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape51: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape50, gv442, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape50) model_layers_12_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[74] gv443: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc310: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv443, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape51, model_layers_12_self_attn_o_proj_weight1, alloc310) R.vm.kill_object(reshape51) R.vm.kill_object(model_layers_12_self_attn_o_proj_weight1) model_layers_12_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[78] gv444: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc311: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv444, R.dtype("float16")) gv445: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc312: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv445, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc310, alloc307, model_layers_12_post_attention_layernorm_weight1, alloc311, alloc312) R.vm.kill_object(alloc307) R.vm.kill_object(alloc310) R.vm.kill_object(model_layers_12_post_attention_layernorm_weight1) lv336: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc311, alloc312 model_layers_12_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[75] gv446: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc313: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv446, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc311, model_layers_12_mlp_gate_up_proj_weight1, alloc313) R.vm.kill_object(alloc311) R.vm.kill_object(model_layers_12_mlp_gate_up_proj_weight1) gv447: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc314: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv447, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc313, alloc314) R.vm.kill_object(alloc313) model_layers_12_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[76] gv448: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc315: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv448, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc314, model_layers_12_mlp_down_proj_weight1, alloc315) R.vm.kill_object(alloc314) R.vm.kill_object(model_layers_12_mlp_down_proj_weight1) model_layers_13_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[83] gv449: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc316: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv449, R.dtype("float16")) gv450: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc317: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv450, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc315, alloc312, model_layers_13_input_layernorm_weight1, alloc316, alloc317) R.vm.kill_object(alloc312) R.vm.kill_object(alloc315) R.vm.kill_object(model_layers_13_input_layernorm_weight1) lv338: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc316, alloc317 model_layers_13_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[79] gv451: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc318: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv451, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc316, model_layers_13_self_attn_qkv_proj_weight1, alloc318) R.vm.kill_object(alloc316) R.vm.kill_object(model_layers_13_self_attn_qkv_proj_weight1) gv452: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape52: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc318, gv452, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc318) gv453: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape53: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape52, gv453, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape52) gv454: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc319: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv454, R.dtype("float16")) _256: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(13), R.prim_value(T.float32(1.0)), reshape53, alloc319) R.vm.kill_object(reshape53) gv455: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape54: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc319, gv455, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc319) gv456: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape55: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape54, gv456, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape54) model_layers_13_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[80] gv457: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc320: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv457, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape55, model_layers_13_self_attn_o_proj_weight1, alloc320) R.vm.kill_object(reshape55) R.vm.kill_object(model_layers_13_self_attn_o_proj_weight1) model_layers_13_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[84] gv458: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc321: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv458, R.dtype("float16")) gv459: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc322: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv459, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc320, alloc317, model_layers_13_post_attention_layernorm_weight1, alloc321, alloc322) R.vm.kill_object(alloc317) R.vm.kill_object(alloc320) R.vm.kill_object(model_layers_13_post_attention_layernorm_weight1) lv340: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc321, alloc322 model_layers_13_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[81] gv460: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc323: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv460, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc321, model_layers_13_mlp_gate_up_proj_weight1, alloc323) R.vm.kill_object(alloc321) R.vm.kill_object(model_layers_13_mlp_gate_up_proj_weight1) gv461: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc324: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv461, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc323, alloc324) R.vm.kill_object(alloc323) model_layers_13_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[82] gv462: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc325: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv462, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc324, model_layers_13_mlp_down_proj_weight1, alloc325) R.vm.kill_object(alloc324) R.vm.kill_object(model_layers_13_mlp_down_proj_weight1) model_layers_14_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[89] gv463: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc326: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv463, R.dtype("float16")) gv464: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc327: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv464, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc325, alloc322, model_layers_14_input_layernorm_weight1, alloc326, alloc327) R.vm.kill_object(alloc322) R.vm.kill_object(alloc325) R.vm.kill_object(model_layers_14_input_layernorm_weight1) lv342: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc326, alloc327 model_layers_14_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[85] gv465: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc328: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv465, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc326, model_layers_14_self_attn_qkv_proj_weight1, alloc328) R.vm.kill_object(alloc326) R.vm.kill_object(model_layers_14_self_attn_qkv_proj_weight1) gv466: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape56: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc328, gv466, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc328) gv467: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape57: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape56, gv467, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape56) gv468: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc329: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv468, R.dtype("float16")) _264: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(14), R.prim_value(T.float32(1.0)), reshape57, alloc329) R.vm.kill_object(reshape57) gv469: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape58: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc329, gv469, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc329) gv470: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape59: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape58, gv470, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape58) model_layers_14_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[86] gv471: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc330: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv471, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape59, model_layers_14_self_attn_o_proj_weight1, alloc330) R.vm.kill_object(reshape59) R.vm.kill_object(model_layers_14_self_attn_o_proj_weight1) model_layers_14_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[90] gv472: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc331: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv472, R.dtype("float16")) gv473: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc332: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv473, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc330, alloc327, model_layers_14_post_attention_layernorm_weight1, alloc331, alloc332) R.vm.kill_object(alloc327) R.vm.kill_object(alloc330) R.vm.kill_object(model_layers_14_post_attention_layernorm_weight1) lv344: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc331, alloc332 model_layers_14_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[87] gv474: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc333: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv474, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc331, model_layers_14_mlp_gate_up_proj_weight1, alloc333) R.vm.kill_object(alloc331) R.vm.kill_object(model_layers_14_mlp_gate_up_proj_weight1) gv475: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc334: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv475, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc333, alloc334) R.vm.kill_object(alloc333) model_layers_14_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[88] gv476: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc335: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv476, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc334, model_layers_14_mlp_down_proj_weight1, alloc335) R.vm.kill_object(alloc334) R.vm.kill_object(model_layers_14_mlp_down_proj_weight1) model_layers_15_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[95] gv477: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc336: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv477, R.dtype("float16")) gv478: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc337: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv478, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc335, alloc332, model_layers_15_input_layernorm_weight1, alloc336, alloc337) R.vm.kill_object(alloc332) R.vm.kill_object(alloc335) R.vm.kill_object(model_layers_15_input_layernorm_weight1) lv346: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc336, alloc337 model_layers_15_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[91] gv479: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc338: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv479, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc336, model_layers_15_self_attn_qkv_proj_weight1, alloc338) R.vm.kill_object(alloc336) R.vm.kill_object(model_layers_15_self_attn_qkv_proj_weight1) gv480: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape60: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc338, gv480, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc338) gv481: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape61: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape60, gv481, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape60) gv482: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc339: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv482, R.dtype("float16")) _272: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(15), R.prim_value(T.float32(1.0)), reshape61, alloc339) R.vm.kill_object(reshape61) gv483: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape62: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc339, gv483, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc339) gv484: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape63: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape62, gv484, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape62) model_layers_15_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[92] gv485: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc340: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv485, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape63, model_layers_15_self_attn_o_proj_weight1, alloc340) R.vm.kill_object(reshape63) R.vm.kill_object(model_layers_15_self_attn_o_proj_weight1) model_layers_15_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[96] gv486: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc341: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv486, R.dtype("float16")) gv487: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc342: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv487, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc340, alloc337, model_layers_15_post_attention_layernorm_weight1, alloc341, alloc342) R.vm.kill_object(alloc337) R.vm.kill_object(alloc340) R.vm.kill_object(model_layers_15_post_attention_layernorm_weight1) lv348: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc341, alloc342 model_layers_15_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[93] gv488: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc343: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv488, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc341, model_layers_15_mlp_gate_up_proj_weight1, alloc343) R.vm.kill_object(alloc341) R.vm.kill_object(model_layers_15_mlp_gate_up_proj_weight1) gv489: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc344: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv489, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc343, alloc344) R.vm.kill_object(alloc343) model_layers_15_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[94] gv490: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc345: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv490, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc344, model_layers_15_mlp_down_proj_weight1, alloc345) R.vm.kill_object(alloc344) R.vm.kill_object(model_layers_15_mlp_down_proj_weight1) model_layers_16_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[101] gv491: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc346: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv491, R.dtype("float16")) gv492: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc347: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv492, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc345, alloc342, model_layers_16_input_layernorm_weight1, alloc346, alloc347) R.vm.kill_object(alloc342) R.vm.kill_object(alloc345) R.vm.kill_object(model_layers_16_input_layernorm_weight1) lv350: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc346, alloc347 model_layers_16_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[97] gv493: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc348: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv493, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc346, model_layers_16_self_attn_qkv_proj_weight1, alloc348) R.vm.kill_object(alloc346) R.vm.kill_object(model_layers_16_self_attn_qkv_proj_weight1) gv494: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape64: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc348, gv494, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc348) gv495: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape65: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape64, gv495, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape64) gv496: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc349: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv496, R.dtype("float16")) _280: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(16), R.prim_value(T.float32(1.0)), reshape65, alloc349) R.vm.kill_object(reshape65) gv497: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape66: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc349, gv497, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc349) gv498: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape67: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape66, gv498, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape66) model_layers_16_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[98] gv499: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc350: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv499, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape67, model_layers_16_self_attn_o_proj_weight1, alloc350) R.vm.kill_object(reshape67) R.vm.kill_object(model_layers_16_self_attn_o_proj_weight1) model_layers_16_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[102] gv500: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc351: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv500, R.dtype("float16")) gv501: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc352: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv501, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc350, alloc347, model_layers_16_post_attention_layernorm_weight1, alloc351, alloc352) R.vm.kill_object(alloc347) R.vm.kill_object(alloc350) R.vm.kill_object(model_layers_16_post_attention_layernorm_weight1) lv352: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc351, alloc352 model_layers_16_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[99] gv502: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc353: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv502, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc351, model_layers_16_mlp_gate_up_proj_weight1, alloc353) R.vm.kill_object(alloc351) R.vm.kill_object(model_layers_16_mlp_gate_up_proj_weight1) gv503: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc354: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv503, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc353, alloc354) R.vm.kill_object(alloc353) model_layers_16_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[100] gv504: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc355: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv504, R.dtype("float16")) cls.gemma_q0f16_NT_matmul8(alloc354, model_layers_16_mlp_down_proj_weight1, alloc355) R.vm.kill_object(alloc354) R.vm.kill_object(model_layers_16_mlp_down_proj_weight1) model_layers_17_input_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[107] gv505: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc356: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv505, R.dtype("float16")) gv506: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc357: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv506, R.dtype("float16")) cls.gemma_q0f16_fuse_add_norm_prefill(alloc355, alloc352, model_layers_17_input_layernorm_weight1, alloc356, alloc357) R.vm.kill_object(alloc352) R.vm.kill_object(alloc355) R.vm.kill_object(model_layers_17_input_layernorm_weight1) lv354: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc356, alloc357 model_layers_17_self_attn_qkv_proj_weight1: R.Tensor((2560, 2048), dtype="float16") = packed_params[103] gv507: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2560), sinfo_args=(R.Shape(ndim=3),)) alloc358: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv507, R.dtype("float16")) cls.gemma_q0f16_NT_matmul5(alloc356, model_layers_17_self_attn_qkv_proj_weight1, alloc358) R.vm.kill_object(alloc356) R.vm.kill_object(model_layers_17_self_attn_qkv_proj_weight1) gv508: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape68: R.Tensor((1, seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc358, gv508, sinfo_args=(R.Tensor((1, seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(alloc358) gv509: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(10), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) reshape69: R.Tensor((seq_len, 10, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape68, gv509, sinfo_args=(R.Tensor((seq_len, 10, 256), dtype="float16"),)) R.vm.kill_object(reshape68) gv510: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=3),)) alloc359: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv510, R.dtype("float16")) _288: R.Object = R.call_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", paged_kv_cache, R.prim_value(17), R.prim_value(T.float32(1.0)), reshape69, alloc359) R.vm.kill_object(reshape69) gv511: R.Shape(ndim=4) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(4), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(8), R.prim_value(0), R.prim_value(256), sinfo_args=(R.Shape(ndim=4),)) reshape70: R.Tensor((1, seq_len, 8, 256), dtype="float16") = R.call_packed("vm.builtin.reshape", alloc359, gv511, sinfo_args=(R.Tensor((1, seq_len, 8, 256), dtype="float16"),)) R.vm.kill_object(alloc359) gv512: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) reshape71: R.Tensor((1, seq_len, 2048), dtype="float16") = R.call_packed("vm.builtin.reshape", reshape70, gv512, sinfo_args=(R.Tensor((1, seq_len, 2048), dtype="float16"),)) R.vm.kill_object(reshape70) model_layers_17_self_attn_o_proj_weight1: R.Tensor((2048, 2048), dtype="float16") = packed_params[104] gv513: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc360: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv513, R.dtype("float16")) cls.gemma_q0f16_NT_matmul6(reshape71, model_layers_17_self_attn_o_proj_weight1, alloc360) R.vm.kill_object(reshape71) R.vm.kill_object(model_layers_17_self_attn_o_proj_weight1) model_layers_17_post_attention_layernorm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[108] gv514: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc361: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv514, R.dtype("float16")) gv515: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc362: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage9, R.prim_value(0), gv515, R.dtype("float16")) R.vm.kill_object(storage9) cls.gemma_q0f16_fuse_add_norm_prefill(alloc360, alloc357, model_layers_17_post_attention_layernorm_weight1, alloc361, alloc362) R.vm.kill_object(alloc357) R.vm.kill_object(alloc360) R.vm.kill_object(model_layers_17_post_attention_layernorm_weight1) lv356: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc361, alloc362 model_layers_17_mlp_gate_up_proj_weight1: R.Tensor((32768, 2048), dtype="float16") = packed_params[105] gv516: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(32768), sinfo_args=(R.Shape(ndim=3),)) alloc363: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv516, R.dtype("float16")) cls.gemma_q0f16_NT_matmul7(alloc361, model_layers_17_mlp_gate_up_proj_weight1, alloc363) R.vm.kill_object(alloc361) R.vm.kill_object(model_layers_17_mlp_gate_up_proj_weight1) gv517: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(16384), sinfo_args=(R.Shape(ndim=3),)) alloc364: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv517, R.dtype("float16")) cls.gemma_q0f16_fused_split1_gelu_tanh1_multiply3(alloc363, alloc364) R.vm.kill_object(alloc363) model_layers_17_mlp_down_proj_weight1: R.Tensor((2048, 16384), dtype="float16") = packed_params[106] gv518: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc365: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage8, R.prim_value(0), gv518, R.dtype("float16")) R.vm.kill_object(storage8) cls.gemma_q0f16_NT_matmul8(alloc364, model_layers_17_mlp_down_proj_weight1, alloc365) R.vm.kill_object(alloc364) R.vm.kill_object(model_layers_17_mlp_down_proj_weight1) model_norm_weight1: R.Tensor((2048,), dtype="float16") = packed_params[109] gv519: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc366: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage11, R.prim_value(0), gv519, R.dtype("float16")) R.vm.kill_object(storage11) gv520: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(2048), sinfo_args=(R.Shape(ndim=3),)) alloc367: R.Tensor(dtype="float16", ndim=3) = R.vm.alloc_tensor(storage10, R.prim_value(0), gv520, R.dtype("float16")) R.vm.kill_object(storage10) cls.gemma_q0f16_fuse_add_norm_prefill(alloc365, alloc362, model_norm_weight1, alloc366, alloc367) R.vm.kill_object(alloc362) R.vm.kill_object(alloc365) R.vm.kill_object(model_norm_weight1) lv358: R.Tuple(R.Tensor(dtype="float16", ndim=3), R.Tensor(dtype="float16", ndim=3)) = alloc366, alloc367 R.vm.kill_object(alloc367) storage12: R.Object = R.vm.alloc_storage(R.shape([4096]), R.prim_value(0), R.dtype("uint8"), R.str("global")) alloc368: R.Tensor((1, 1, 2048), dtype="float16") = R.vm.alloc_tensor(storage12, R.prim_value(0), R.shape([1, 1, 2048]), R.dtype("float16")) R.vm.kill_object(storage12) cls.gemma_q0f16_index(alloc366, alloc368) R.vm.kill_object(alloc366) model_embed_tokens_weight1: R.Tensor((vocab_size, 2048), dtype="float16") = packed_params[0] gv521: R.Shape(ndim=1) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(3), sinfo_args=(R.Shape(ndim=1),)) storage13: R.Object = R.vm.alloc_storage(gv521, R.prim_value(0), R.dtype("uint8"), R.str("global")) gv522: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(1), sinfo_args=(R.Shape(ndim=3),)) alloc369: R.Tensor(dtype="float32", ndim=3) = R.vm.alloc_tensor(storage13, R.prim_value(0), gv522, R.dtype("float32")) R.vm.kill_object(storage13) cls.gemma_q0f16_NT_matmul14(alloc368, model_embed_tokens_weight1, alloc369) R.vm.kill_object(alloc368) R.vm.kill_object(model_embed_tokens_weight1) gv1: R.Tuple(R.Tensor(dtype="float32", ndim=3), R.Object) = alloc369, paged_kv_cache R.vm.kill_object(alloc369) gv523: R.Tensor(dtype="float32", ndim=3) = gv1[0] R.call_packed("vm.builtin.match_shape", gv523, shape_heap, R.prim_value(3), R.prim_value(0), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(3), R.prim_value(1), R.str("ErrorContext(fn=prefill, loc=return, annotation=R.Tuple(R.Tensor((1, 1, vocab_size), dtype=\"float32\"), R.Object)) "), sinfo_args=(R.Tuple,)) return gv1 @R.function def softmax_with_temperature(logits: R.Tensor(("batch_size", 1, "vocab_size"), dtype="float32"), temperature: R.Tensor(("batch_size",), dtype="float32")) -> R.Tensor(("batch_size", 1, "vocab_size"), dtype="float32"): batch_size = T.int64(is_size_var=True) vocab_size = T.int64(is_size_var=True) R.func_attr({"relax.force_pure": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 8192, "total_seq_len": 8192}}) cls = Module shape_heap: R.Tensor(dtype="int64", ndim=1) = R.call_builtin_with_ctx("vm.builtin.alloc_shape_heap", (R.prim_value(5),), sinfo_args=(R.Tensor(dtype="int64", ndim=1),)) R.call_packed("vm.builtin.check_tensor_info", logits, R.prim_value(3), R.dtype("float32"), R.str("ErrorContext(fn=softmax_with_temperature, loc=param[0], param=logits, annotation=R.Tensor((batch_size, 1, vocab_size), dtype=\"float32\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.check_tensor_info", temperature, R.prim_value(1), R.dtype("float32"), R.str("ErrorContext(fn=softmax_with_temperature, loc=param[1], param=temperature, annotation=R.Tensor((batch_size,), dtype=\"float32\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", logits, shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(1), R.str("ErrorContext(fn=softmax_with_temperature, loc=param[0], param=logits, annotation=R.Tensor((batch_size, 1, vocab_size), dtype=\"float32\")) "), sinfo_args=(R.Tuple,)) R.call_packed("vm.builtin.match_shape", temperature, shape_heap, R.prim_value(1), R.prim_value(3), R.prim_value(0), R.str("ErrorContext(fn=softmax_with_temperature, loc=param[1], param=temperature, annotation=R.Tensor((batch_size,), dtype=\"float32\")) "), sinfo_args=(R.Tuple,)) cls.gemma_q0f16_shape_func(shape_heap) gv: R.Shape(ndim=2) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(1), sinfo_args=(R.Shape(ndim=2),)) lv: R.Tensor((batch_size, vocab_size), dtype="float32") = R.call_packed("vm.builtin.reshape", logits, gv, sinfo_args=(R.Tensor((batch_size, vocab_size), dtype="float32"),)) gv1: R.Shape(ndim=1) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(2), sinfo_args=(R.Shape(ndim=1),)) storage: R.Object = R.vm.alloc_storage(gv1, R.prim_value(0), R.dtype("uint8"), R.str("global")) gv2: R.Shape(ndim=2) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(3), sinfo_args=(R.Shape(ndim=2),)) alloc: R.Tensor(dtype="float32", ndim=2) = R.vm.alloc_tensor(storage, R.prim_value(0), gv2, R.dtype("float32")) R.vm.kill_object(storage) gv3: R.Shape(ndim=1) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(2), sinfo_args=(R.Shape(ndim=1),)) storage1: R.Object = R.vm.alloc_storage(gv3, R.prim_value(0), R.dtype("uint8"), R.str("global")) gv4: R.Shape(ndim=2) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(3), sinfo_args=(R.Shape(ndim=2),)) alloc1: R.Tensor(dtype="float32", ndim=2) = R.vm.alloc_tensor(storage1, R.prim_value(0), gv4, R.dtype("float32")) R.vm.kill_object(storage1) cls.gemma_q0f16_chunk_lse(lv, temperature, alloc, alloc1) lv1: R.Tuple(R.Tensor(dtype="float32", ndim=2), R.Tensor(dtype="float32", ndim=2)) = alloc, alloc1 gv5: R.Shape(ndim=1) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(1), R.prim_value(1), R.prim_value(4), sinfo_args=(R.Shape(ndim=1),)) storage2: R.Object = R.vm.alloc_storage(gv5, R.prim_value(0), R.dtype("uint8"), R.str("global")) gv6: R.Shape(ndim=2) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(2), R.prim_value(1), R.prim_value(0), R.prim_value(1), R.prim_value(1), sinfo_args=(R.Shape(ndim=2),)) alloc2: R.Tensor(dtype="float32", ndim=2) = R.vm.alloc_tensor(storage2, R.prim_value(0), gv6, R.dtype("float32")) R.vm.kill_object(storage2) cls.gemma_q0f16_softmax_with_chunked_sum(lv, temperature, alloc, alloc1, alloc2) R.vm.kill_object(lv) R.vm.kill_object(alloc) R.vm.kill_object(alloc1) gv7: R.Shape(ndim=3) = R.call_packed("vm.builtin.make_shape", shape_heap, R.prim_value(3), R.prim_value(1), R.prim_value(0), R.prim_value(0), R.prim_value(1), R.prim_value(1), R.prim_value(1), sinfo_args=(R.Shape(ndim=3),)) gv_1: R.Tensor((batch_size, 1, vocab_size), dtype="float32") = R.call_packed("vm.builtin.reshape", alloc2, gv7, sinfo_args=(R.Tensor((batch_size, 1, vocab_size), dtype="float32"),)) R.vm.kill_object(alloc2) return gv_1