Download ir_dump/debug-phase0.py from AMAI-Lab/pythia-70m-q4f16-1: direct link, hf CLI and curl.
- Browser
- Download file 411 kB
-
https://huggingface.co/AMAI-Lab/pythia-70m-q4f16-1/resolve/main/ir_dump/debug-phase0.py
- Command line
-
hf download hf://AMAI-Lab/pythia-70m-q4f16-1/ir_dump/debug-phase0.py
-
curl -L -o debug-phase0.py https://huggingface.co/AMAI-Lab/pythia-70m-q4f16-1/resolve/main/ir_dump/debug-phase0.py
411 kB
# from tvm.script import ir as I
# from tvm.script import tir as T
# from tvm.script import relax as R
@I.ir_module
class Module:
I.module_attrs({"system_lib_prefix": "gpt_neox_q4f16_1_"})
@T.prim_func
def apply_bitmask_inplace(var_logits: T.handle, var_seq_ids: T.handle, var_bitmask: T.handle):
T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": T.bool(True), "tir.noalias": T.bool(True)})
batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True)
logits = T.match_buffer(var_logits, (batch_size, vocab_size))
num_seq = T.int32(is_size_var=True)
seq_ids = T.match_buffer(var_seq_ids, (num_seq,), "int32")
bitmask = T.match_buffer(var_bitmask, (batch_size, (vocab_size + 31) // 32), "int32")
# with T.block("root"):
for fused_s_v_0 in T.thread_binding((num_seq * vocab_size + 255) // 256, thread="blockIdx.x"):
for fused_s_v_1 in T.thread_binding(256, thread="threadIdx.x"):
with T.block("block"):
vs = T.axis.spatial(num_seq, (fused_s_v_0 * 256 + fused_s_v_1) // vocab_size)
vv = T.axis.spatial(vocab_size, (fused_s_v_0 * 256 + fused_s_v_1) % vocab_size)
T.where(fused_s_v_0 * 256 + fused_s_v_1 < num_seq * vocab_size)
T.reads(bitmask[seq_ids[vs], vv // 32], seq_ids[vs], logits[seq_ids[vs], vv])
T.writes(logits[seq_ids[vs], vv])
logits[seq_ids[vs], vv] = T.if_then_else(T.bitwise_and(T.shift_right(bitmask[seq_ids[vs], vv // 32], vv % 32), 1) == 1, logits[seq_ids[vs], vv], T.float32(-340282346638528859811704183484516925440.0))
@T.prim_func
def apply_logit_bias_inplace(var_logits: T.handle, var_pos2seq_id: T.handle, var_token_ids: T.handle, var_logit_bias: T.handle):
T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": T.bool(True), "tir.noalias": T.bool(True)})
batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True)
logits = T.match_buffer(var_logits, (batch_size, vocab_size))
num_token = T.int32(is_size_var=True)
pos2seq_id = T.match_buffer(var_pos2seq_id, (num_token,), "int32")
token_ids = T.match_buffer(var_token_ids, (num_token,), "int32")
logit_bias = T.match_buffer(var_logit_bias, (num_token,))
# with T.block("root"):
for p0 in T.thread_binding((num_token + 255) // 256, thread="blockIdx.x"):
for p1 in T.thread_binding(256, thread="threadIdx.x"):
with T.block("block"):
vp = T.axis.spatial(num_token, p0 * 256 + p1)
T.where(p0 * 256 + p1 < num_token)
T.reads(logits[pos2seq_id[vp], token_ids[vp]], pos2seq_id[vp], token_ids[vp], logit_bias[vp])
T.writes(logits[pos2seq_id[vp], token_ids[vp]])
logits[pos2seq_id[vp], token_ids[vp]] = logits[pos2seq_id[vp], token_ids[vp]] + logit_bias[vp]
@T.prim_func
def apply_penalty_inplace(var_logits: T.handle, var_seq_ids: T.handle, var_pos2seq_id: T.handle, var_token_ids: T.handle, var_token_cnt: T.handle, var_penalties: T.handle):
T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": T.bool(True), "tir.noalias": T.bool(True)})
batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True)
logits = T.match_buffer(var_logits, (batch_size, vocab_size))
num_seq = T.int32(is_size_var=True)
seq_ids = T.match_buffer(var_seq_ids, (num_seq,), "int32")
num_token = T.int32(is_size_var=True)
pos2seq_id = T.match_buffer(var_pos2seq_id, (num_token,), "int32")
token_ids = T.match_buffer(var_token_ids, (num_token,), "int32")
token_cnt = T.match_buffer(var_token_cnt, (num_token,), "int32")
penalties = T.match_buffer(var_penalties, (num_seq, 3))
# with T.block("root"):
for p0 in T.thread_binding((num_token + 255) // 256, thread="blockIdx.x"):
for p1 in T.thread_binding(256, thread="threadIdx.x"):
with T.block("block"):
vp = T.axis.spatial(num_token, p0 * 256 + p1)
T.where(p0 * 256 + p1 < num_token)
T.reads(logits[seq_ids[pos2seq_id[vp]], token_ids[vp]], seq_ids[pos2seq_id[vp]], pos2seq_id[vp], token_ids[vp], penalties[pos2seq_id[vp], 0:3], token_cnt[vp])
T.writes(logits[seq_ids[pos2seq_id[vp]], token_ids[vp]])
logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] = logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] - (penalties[pos2seq_id[vp], 0] + T.Cast("float32", token_cnt[vp]) * penalties[pos2seq_id[vp], 1])
logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] = T.if_then_else(logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] < T.float32(0.0), logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] * penalties[pos2seq_id[vp], 2], logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] / penalties[pos2seq_id[vp], 2])
@T.prim_func
def batch_decode_paged_kv(_0: T.int32, Q_handle: T.handle, pages_handle: T.handle, page_table_indptr_handle: T.handle, page_table_values_handle: T.handle, var_length_info: T.handle, k_rope_pos_offset_handle: T.handle, q_rope_position_handle: T.handle, output_handle: T.handle, lse_handle: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32):
T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1})
B = T.int32(is_size_var=True)
Q = T.match_buffer(Q_handle, (B, 8, 64), "float16")
max_num_pages = T.int32(is_size_var=True)
pages = T.match_buffer(pages_handle, (max_num_pages, 2, 8, 16, 64), "float16", offset_factor=1)
page_table_indptr = T.match_buffer(page_table_indptr_handle, (B + 1,), "int32", offset_factor=1)
nnz_pages = T.int32(is_size_var=True)
page_table_values = T.match_buffer(page_table_values_handle, (nnz_pages,), "int32", offset_factor=1)
length_info = T.match_buffer(var_length_info, (B,), "int32", offset_factor=1)
k_rope_pos_offset = T.match_buffer(k_rope_pos_offset_handle, (B,), "int32", offset_factor=1)
q_rope_position = T.match_buffer(q_rope_position_handle, (B,), "int32", offset_factor=1)
output = T.match_buffer(output_handle, (B, 8, 64), "float16")
lse = T.match_buffer(lse_handle, (B, 8))
# with T.block("root"):
sm_scale: T.float32 = T.float32(0.18033688011112042)
for bx in T.thread_binding(B, thread="blockIdx.x"):
for fused_by_bz in T.thread_binding(8, thread="blockIdx.y"):
for ty in T.thread_binding(1, thread="threadIdx.y"):
for tx in T.thread_binding(16, thread="threadIdx.x"):
for tz in T.thread_binding(16, thread="threadIdx.z"):
with T.block("attn"):
T.reads(page_table_indptr[bx:bx + 2], length_info[bx], q_rope_position[bx], Q[bx, fused_by_bz // 8 + ty + fused_by_bz % 8, tx * 4 - 32:tx * 4 - 32 + 68])
T.writes(output[bx, fused_by_bz % 8 + fused_by_bz // 8 + ty, tx * 4:tx * 4 + 4], lse[bx, fused_by_bz % 8 + fused_by_bz // 8 + ty])
Q_local = T.alloc_buffer((4,), "float16", scope="local")
kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local")
K_smem = T.alloc_buffer((16, 64), "float16", scope="shared")
V_smem = T.alloc_buffer((16, 64), "float16", scope="shared")
O_allreduce = T.alloc_buffer((16, 1, 64), scope="shared")
md_allreduce = T.alloc_buffer((16, 1, 2), scope="shared")
S_reduce_local = T.alloc_buffer((1,), scope="local")
t0 = T.alloc_buffer((1,), scope="local")
S_local = T.alloc_buffer((1,), scope="local")
QK_local = T.alloc_buffer((4,), scope="local")
V_local = T.alloc_buffer((4,), "float16", scope="local")
m_prev = T.alloc_buffer((1,), scope="local")
d_prev = T.alloc_buffer((1,), scope="local")
other_m = T.alloc_buffer((1,), scope="local")
other_d = T.alloc_buffer((1,), scope="local")
exp_mprev = T.alloc_buffer((1,), scope="local")
exp_otherm = T.alloc_buffer((1,), scope="local")
other_o = T.alloc_buffer((4,), scope="local")
st_m = T.alloc_buffer((1,), scope="local")
st_d = T.alloc_buffer((1,), scope="local")
O_local = T.alloc_buffer((4,), scope="local")
by: T.int32 = fused_by_bz % 8
bz: T.int32 = fused_by_bz // 8
batch_idx: T.int32 = bx
cur_page_indptr_begin: T.int32 = page_table_indptr[batch_idx]
cur_page_indptr_end: T.int32 = page_table_indptr[batch_idx + 1]
kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[batch_idx], 0)
st_m[0] = T.float32(-50000.0)
st_d[0] = T.float32(1.0)
for vec in T.vectorized(4):
O_local[vec] = T.float32(0.0)
for vec in T.vectorized(4):
freq = T.float32()
Q_local[vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", Q[bx, by + bz + ty, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 32, Q[bx, by + bz + ty, tx * 4 + vec + 32] * T.float16(-1.0), Q[bx, by + bz + ty, tx * 4 + vec - 32]))), where={freq: T.Cast("float32", q_rope_position[batch_idx]) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 64) / T.float32(64.0))}), Q[bx, by + bz + ty, tx * 4 + vec])
for iterator in range((kv_chunk_len[0] + 15) // 16):
tile_start_s: T.int32 = tz + ty
tile_start_g: T.int32 = iterator * 16 + tz + ty
for j in range(1):
with T.block("KV_load"):
T.reads()
T.writes()
row_g: T.int32 = tile_start_g + j
if row_g < kv_chunk_len[0]:
seq_offset: T.int32 = row_g
page_no: T.int32 = page_table_values[cur_page_indptr_begin + seq_offset // 16]
page_offset: T.int32 = seq_offset % 16
for vec in T.vectorized(4):
freq = T.float32()
K_smem[tile_start_s + j, tx * 4 + vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 32, pages[page_no, 0, by, page_offset, tx * 4 + vec + 32] * T.float16(-1.0), pages[page_no, 0, by, page_offset, tx * 4 + vec - 32]))), where={freq: T.Cast("float32", k_rope_pos_offset[batch_idx] + row_g) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 64) / T.float32(64.0))}), pages[page_no, 0, by, page_offset, tx * 4 + vec])
V_smem[tile_start_s + j, tx * 4 + vec] = pages[page_no, 1, by, page_offset, tx * 4 + vec]
else:
for vec in T.vectorized(4):
K_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0)
V_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0)
T.tvm_storage_sync("shared")
m_prev[0] = st_m[0]
for j in range(1):
for vec in T.vectorized(4):
QK_local[vec] = T.Cast("float32", Q_local[vec]) * T.Cast("float32", K_smem[tz + j, tx * 4 + vec]) * attn_score_scaling_factor * sm_scale
S_reduce_local[0] = T.float32(0.0)
for vec in T.unroll(4):
S_reduce_local[0] = S_reduce_local[0] + QK_local[vec]
with T.block("block_cross_thread"):
T.reads(S_reduce_local[0])
T.writes(t0[0])
T.attr(T.comm_reducer(lambda x0, y0: x0 + y0, [T.float32(0.0)]), "reduce_scope", T.reinterpret("handle", T.uint64(0)))
T.tvm_thread_allreduce(T.uint32(1), S_reduce_local[0], T.bool(True), t0[0], tx)
S_local[j] = T.float32(-50000.0)
if iterator * 16 + tz + j < kv_chunk_len[0]:
S_local[j] = t0[0]
st_m[0] = T.max(st_m[0], S_local[j])
o_scale: T.float32 = T.exp2(m_prev[0] - st_m[0])
st_d[0] = st_d[0] * o_scale
for j in range(1):
S_local[j] = T.exp2(S_local[j] - st_m[0])
st_d[0] = st_d[0] + S_local[j]
for j in T.vectorized(4):
O_local[j] = O_local[j] * o_scale
for j in range(1):
for vec in T.vectorized(4):
V_local[vec] = V_smem[tz + j, tx * 4 + vec]
for vec in T.vectorized(4):
O_local[vec] = O_local[vec] + T.Cast("float32", V_local[vec]) * S_local[j]
for vec in T.vectorized(4):
O_allreduce[tz, ty, tx * 4 + vec] = O_local[vec]
md_allreduce[tz, ty, 0] = st_m[0]
md_allreduce[tz, ty, 1] = st_d[0]
T.tvm_storage_sync("shared")
st_m[0] = T.float32(-50000.0)
st_d[0] = T.float32(1.0)
for vec in T.vectorized(4):
O_local[vec] = T.float32(0.0)
for j in range(16):
m_prev[0] = st_m[0]
d_prev[0] = st_d[0]
other_m[0] = md_allreduce[j, ty, 0]
other_d[0] = md_allreduce[j, ty, 1]
for vec in T.vectorized(4):
other_o[vec] = O_allreduce[j, ty, tx * 4 + vec]
st_m[0] = T.max(st_m[0], other_m[0])
st_d[0] = d_prev[0] * T.exp2(m_prev[0] - st_m[0]) + other_d[0] * T.exp2(other_m[0] - st_m[0])
exp_mprev[0] = T.exp2(m_prev[0] - st_m[0])
exp_otherm[0] = T.exp2(other_m[0] - st_m[0])
for vec in T.vectorized(4):
O_local[vec] = O_local[vec] * exp_mprev[0] + other_o[vec] * exp_otherm[0]
for vec in T.vectorized(4):
O_local[vec] = O_local[vec] / st_d[0]
for vec in T.vectorized(4):
output[batch_idx, by + bz + ty, tx * 4 + vec] = T.Cast("float16", O_local[vec])
lse[batch_idx, by + bz + ty] = st_m[0] + T.log2(st_d[0])
@T.prim_func
def batch_decode_paged_kv_sliding_window(_0: T.int32, Q_handle: T.handle, pages_handle: T.handle, page_table_indptr_handle: T.handle, page_table_values_handle: T.handle, var_length_info: T.handle, k_rope_pos_offset_handle: T.handle, q_rope_position_handle: T.handle, output_handle: T.handle, lse_handle: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32):
T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1})
B = T.int32(is_size_var=True)
Q = T.match_buffer(Q_handle, (B, 8, 64), "float16")
max_num_pages = T.int32(is_size_var=True)
pages = T.match_buffer(pages_handle, (max_num_pages, 2, 8, 16, 64), "float16", offset_factor=1)
page_table_indptr = T.match_buffer(page_table_indptr_handle, (B + 1,), "int32", offset_factor=1)
nnz_pages = T.int32(is_size_var=True)
page_table_values = T.match_buffer(page_table_values_handle, (nnz_pages,), "int32", offset_factor=1)
length_info = T.match_buffer(var_length_info, (3, B), "int32", offset_factor=1)
k_rope_pos_offset = T.match_buffer(k_rope_pos_offset_handle, (B,), "int32", offset_factor=1)
q_rope_position = T.match_buffer(q_rope_position_handle, (B,), "int32", offset_factor=1)
output = T.match_buffer(output_handle, (B, 8, 64), "float16")
lse = T.match_buffer(lse_handle, (B, 8))
# with T.block("root"):
sm_scale: T.float32 = T.float32(0.18033688011112042)
for bx in T.thread_binding(B, thread="blockIdx.x"):
for fused_by_bz in T.thread_binding(8, thread="blockIdx.y"):
for ty in T.thread_binding(1, thread="threadIdx.y"):
for tx in T.thread_binding(16, thread="threadIdx.x"):
for tz in T.thread_binding(16, thread="threadIdx.z"):
with T.block("attn"):
T.reads(page_table_indptr[bx:bx + 2], length_info[0:3, bx], q_rope_position[bx], Q[bx, fused_by_bz // 8 + ty + fused_by_bz % 8, tx * 4 - 32:tx * 4 - 32 + 68])
T.writes(output[bx, fused_by_bz % 8 + fused_by_bz // 8 + ty, tx * 4:tx * 4 + 4], lse[bx, fused_by_bz % 8 + fused_by_bz // 8 + ty])
Q_local = T.alloc_buffer((4,), "float16", scope="local")
kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local")
K_smem = T.alloc_buffer((16, 64), "float16", scope="shared")
V_smem = T.alloc_buffer((16, 64), "float16", scope="shared")
O_allreduce = T.alloc_buffer((16, 1, 64), scope="shared")
md_allreduce = T.alloc_buffer((16, 1, 2), scope="shared")
S_reduce_local = T.alloc_buffer((1,), scope="local")
t0 = T.alloc_buffer((1,), scope="local")
S_local = T.alloc_buffer((1,), scope="local")
QK_local = T.alloc_buffer((4,), scope="local")
V_local = T.alloc_buffer((4,), "float16", scope="local")
m_prev = T.alloc_buffer((1,), scope="local")
d_prev = T.alloc_buffer((1,), scope="local")
other_m = T.alloc_buffer((1,), scope="local")
other_d = T.alloc_buffer((1,), scope="local")
exp_mprev = T.alloc_buffer((1,), scope="local")
exp_otherm = T.alloc_buffer((1,), scope="local")
other_o = T.alloc_buffer((4,), scope="local")
st_m = T.alloc_buffer((1,), scope="local")
st_d = T.alloc_buffer((1,), scope="local")
O_local = T.alloc_buffer((4,), scope="local")
by: T.int32 = fused_by_bz % 8
bz: T.int32 = fused_by_bz // 8
batch_idx: T.int32 = bx
cur_page_indptr_begin: T.int32 = page_table_indptr[batch_idx]
cur_page_indptr_end: T.int32 = page_table_indptr[batch_idx + 1]
kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[0, batch_idx] - length_info[1, batch_idx] + length_info[2, batch_idx], 0)
st_m[0] = T.float32(-50000.0)
st_d[0] = T.float32(1.0)
for vec in T.vectorized(4):
O_local[vec] = T.float32(0.0)
for vec in T.vectorized(4):
freq = T.float32()
Q_local[vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", Q[bx, by + bz + ty, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 32, Q[bx, by + bz + ty, tx * 4 + vec + 32] * T.float16(-1.0), Q[bx, by + bz + ty, tx * 4 + vec - 32]))), where={freq: T.Cast("float32", q_rope_position[batch_idx]) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 64) / T.float32(64.0))}), Q[bx, by + bz + ty, tx * 4 + vec])
for iterator in range((kv_chunk_len[0] + 15) // 16):
tile_start_s: T.int32 = tz + ty
tile_start_g: T.int32 = iterator * 16 + tz + ty
for j in range(1):
with T.block("KV_load"):
T.reads()
T.writes()
row_g: T.int32 = tile_start_g + j
if row_g < kv_chunk_len[0]:
seq_offset: T.int32 = T.if_then_else(row_g < length_info[2, batch_idx], row_g, row_g - length_info[2, batch_idx] + length_info[1, batch_idx])
page_no: T.int32 = page_table_values[cur_page_indptr_begin + seq_offset // 16]
page_offset: T.int32 = seq_offset % 16
for vec in T.vectorized(4):
freq = T.float32()
K_smem[tile_start_s + j, tx * 4 + vec] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, tx * 4 + vec]) + T.sin(freq) * T.Cast("float32", T.if_then_else(tx * 4 + vec < 32, pages[page_no, 0, by, page_offset, tx * 4 + vec + 32] * T.float16(-1.0), pages[page_no, 0, by, page_offset, tx * 4 + vec - 32]))), where={freq: T.Cast("float32", k_rope_pos_offset[batch_idx] + row_g) * rope_scale / T.pow(rope_theta, T.Cast("float32", (tx * 4 + vec) * 2 % 64) / T.float32(64.0))}), pages[page_no, 0, by, page_offset, tx * 4 + vec])
V_smem[tile_start_s + j, tx * 4 + vec] = pages[page_no, 1, by, page_offset, tx * 4 + vec]
else:
for vec in T.vectorized(4):
K_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0)
V_smem[tile_start_s + j, tx * 4 + vec] = T.float16(0.0)
T.tvm_storage_sync("shared")
m_prev[0] = st_m[0]
for j in range(1):
for vec in T.vectorized(4):
QK_local[vec] = T.Cast("float32", Q_local[vec]) * T.Cast("float32", K_smem[tz + j, tx * 4 + vec]) * attn_score_scaling_factor * sm_scale
S_reduce_local[0] = T.float32(0.0)
for vec in T.unroll(4):
S_reduce_local[0] = S_reduce_local[0] + QK_local[vec]
with T.block("block_cross_thread"):
T.reads(S_reduce_local[0])
T.writes(t0[0])
T.attr(T.comm_reducer(lambda x0, y0: x0 + y0, [T.float32(0.0)]), "reduce_scope", T.reinterpret("handle", T.uint64(0)))
T.tvm_thread_allreduce(T.uint32(1), S_reduce_local[0], T.bool(True), t0[0], tx)
S_local[j] = T.float32(-50000.0)
if iterator * 16 + tz + j < kv_chunk_len[0]:
S_local[j] = t0[0]
st_m[0] = T.max(st_m[0], S_local[j])
o_scale: T.float32 = T.exp2(m_prev[0] - st_m[0])
st_d[0] = st_d[0] * o_scale
for j in range(1):
S_local[j] = T.exp2(S_local[j] - st_m[0])
st_d[0] = st_d[0] + S_local[j]
for j in T.vectorized(4):
O_local[j] = O_local[j] * o_scale
for j in range(1):
for vec in T.vectorized(4):
V_local[vec] = V_smem[tz + j, tx * 4 + vec]
for vec in T.vectorized(4):
O_local[vec] = O_local[vec] + T.Cast("float32", V_local[vec]) * S_local[j]
for vec in T.vectorized(4):
O_allreduce[tz, ty, tx * 4 + vec] = O_local[vec]
md_allreduce[tz, ty, 0] = st_m[0]
md_allreduce[tz, ty, 1] = st_d[0]
T.tvm_storage_sync("shared")
st_m[0] = T.float32(-50000.0)
st_d[0] = T.float32(1.0)
for vec in T.vectorized(4):
O_local[vec] = T.float32(0.0)
for j in range(16):
m_prev[0] = st_m[0]
d_prev[0] = st_d[0]
other_m[0] = md_allreduce[j, ty, 0]
other_d[0] = md_allreduce[j, ty, 1]
for vec in T.vectorized(4):
other_o[vec] = O_allreduce[j, ty, tx * 4 + vec]
st_m[0] = T.max(st_m[0], other_m[0])
st_d[0] = d_prev[0] * T.exp2(m_prev[0] - st_m[0]) + other_d[0] * T.exp2(other_m[0] - st_m[0])
exp_mprev[0] = T.exp2(m_prev[0] - st_m[0])
exp_otherm[0] = T.exp2(other_m[0] - st_m[0])
for vec in T.vectorized(4):
O_local[vec] = O_local[vec] * exp_mprev[0] + other_o[vec] * exp_otherm[0]
for vec in T.vectorized(4):
O_local[vec] = O_local[vec] / st_d[0]
for vec in T.vectorized(4):
output[batch_idx, by + bz + ty, tx * 4 + vec] = T.Cast("float16", O_local[vec])
lse[batch_idx, by + bz + ty] = st_m[0] + T.log2(st_d[0])
@T.prim_func
def batch_prefill_paged_kv(_0: T.int32, var_q: T.handle, var_q_indptr: T.handle, var_pages: T.handle, var_page_indptr: T.handle, var_page_values: T.handle, var_length_info: T.handle, var_k_rope_pos_offset: T.handle, var_q_rope_position: T.handle, var_output: T.handle, var_lse: T.handle, causal: T.int32, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32):
T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1})
total_len = T.int32(is_size_var=True)
q = T.match_buffer(var_q, (total_len, 8, 64), "float16")
batch_size = T.int32(is_size_var=True)
q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1)
max_num_pages = T.int32(is_size_var=True)
pages = T.match_buffer(var_pages, (max_num_pages, 2, 8, 16, 64), "float16", offset_factor=1)
page_indptr = T.match_buffer(var_page_indptr, (batch_size + 1,), "int32", offset_factor=1)
nnz_pages = T.int32(is_size_var=True)
page_values = T.match_buffer(var_page_values, (nnz_pages,), "int32", offset_factor=1)
length_info = T.match_buffer(var_length_info, (batch_size,), "int32", offset_factor=1)
k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1)
q_rope_position = T.match_buffer(var_q_rope_position, (total_len,), "int32", offset_factor=1)
output = T.match_buffer(var_output, (total_len, 8, 64), "float16")
lse = T.match_buffer(var_lse, (total_len, 8))
# with T.block("root"):
for lbx in T.thread_binding(16, thread="blockIdx.x"):
for lby in T.thread_binding(8, thread="blockIdx.y"):
for lty in T.thread_binding(4, thread="threadIdx.y"):
for ltx in T.thread_binding(32, thread="threadIdx.x"):
with T.block("attn"):
bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx])
T.reads()
T.writes()
tile_id = T.alloc_buffer((1,), "int32", scope="local")
batch_idx = T.alloc_buffer((1,), "int32", scope="local")
batch_tiles = T.alloc_buffer((1,), "int32", scope="local")
batch_rows = T.alloc_buffer((1,), "int32", scope="local")
iterator = T.alloc_buffer((1,), "int32", scope="local")
kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local")
Q_smem = T.alloc_buffer((32, 64), "float16", scope="shared")
K_smem = T.alloc_buffer((16, 64), "float16", scope="shared")
V_smem = T.alloc_buffer((16, 64), "float16", scope="shared")
S_smem = T.alloc_buffer((32, 16), scope="shared")
S_local = T.alloc_buffer((32, 16), scope="local")
O_local = T.alloc_buffer((32, 64), scope="local")
m_smem = T.alloc_buffer((32,), scope="shared")
m_prev_smem = T.alloc_buffer((32,), scope="shared")
d_smem = T.alloc_buffer((32,), scope="shared")
m_new = T.alloc_buffer((1,), scope="local")
m_prev = T.alloc_buffer((1,), scope="local")
d_new = T.alloc_buffer((1,), scope="local")
tile_id[0] = bx
batch_idx[0] = 0
batch_rows[0] = q_indptr[1] - q_indptr[0]
batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32
while T.tvm_thread_invariant(batch_idx[0] < batch_size):
while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size:
tile_id[0] = tile_id[0] - batch_tiles[0]
batch_idx[0] = batch_idx[0] + 1
if batch_idx[0] < batch_size:
b_idx: T.int32 = batch_idx[0]
batch_rows[0] = q_indptr[b_idx + 1] - q_indptr[b_idx]
batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32
if T.tvm_thread_invariant(batch_idx[0] < batch_size):
b_idx: T.int32 = batch_idx[0]
LH_start: T.int32 = tile_id[0] * 32
q_indptr_val: T.int32 = q_indptr[b_idx]
cur_page_indptr_begin: T.int32 = page_indptr[b_idx]
cur_page_indptr_end: T.int32 = page_indptr[b_idx + 1]
kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[b_idx], 0)
T.tvm_storage_sync("shared")
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
if row < 32:
m_smem[row] = T.float32(-50000.0)
d_smem[row] = T.float32(1.0)
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for li_1, lj_1 in T.grid(4, 4):
with T.block("O_init"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1)
j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1)
T.reads()
T.writes(O_local[i, j])
O_local[i, j] = T.float32(0.0)
T.tvm_storage_sync("shared")
for li_lj_fused_0 in range(4):
for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"):
for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"):
for li_lj_fused_3 in T.vectorized(4):
with T.block("Q_load"):
i = T.axis.spatial(32, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 64)
j = T.axis.spatial(64, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 64)
T.reads()
T.writes()
cur_L: T.int32 = q_indptr_val + (LH_start + i)
cur_H_qo: T.int32 = by
if cur_L < q_indptr[b_idx + 1]:
freq = T.float32()
Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, q[cur_L, cur_H_qo, j + 32] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 32]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), q[cur_L, cur_H_qo, j])
else:
Q_smem[i, j] = T.float16(0.0)
T.tvm_storage_sync("shared")
for iterator_1 in range((kv_chunk_len[0] + 15) // 16):
L_kv_start: T.int32 = iterator_1 * 16
for lz_ly_fused_0 in range(2):
for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"):
for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"):
for lz_ly_fused_3 in T.vectorized(4):
with T.block("K_load"):
i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64)
j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64)
T.reads()
T.writes()
cur_L: T.int32 = L_kv_start + i
if cur_L < kv_chunk_len[0]:
seq_offset: T.int32 = cur_L
page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16]
page_offset: T.int32 = seq_offset % 16
freq = T.float32()
K_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, pages[page_no, 0, by, page_offset, j + 32] * T.float16(-1.0), pages[page_no, 0, by, page_offset, j - 32]))), where={freq: T.Cast("float32", k_rope_pos_offset[b_idx] + cur_L) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), pages[page_no, 0, by, page_offset, j])
else:
K_smem[i, j] = T.float16(0.0)
T.tvm_storage_sync("shared")
for lz_ly_fused_0 in range(2):
for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"):
for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"):
for lz_ly_fused_3 in T.vectorized(4):
with T.block("V_load"):
i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64)
j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64)
T.reads()
T.writes()
cur_L: T.int32 = L_kv_start + i
if cur_L < kv_chunk_len[0]:
seq_offset: T.int32 = cur_L
page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16]
page_offset: T.int32 = seq_offset % 16
V_smem[i, j] = pages[page_no, 1, by, page_offset, j]
else:
V_smem[i, j] = T.float16(0.0)
T.tvm_storage_sync("shared")
with T.block(""):
T.reads(Q_smem[0:32, 0:64], K_smem[0:16, 0:64])
T.writes(S_local[0:32, 0:16])
for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"):
for li_1_init, lj_1_init in T.grid(2, 2):
with T.block("S_gemm_init"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 * 2 + li_1_init)
j = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 2 + lj_1_init)
T.reads()
T.writes(S_local[i, j])
S_local[i, j] = T.float32(0.0)
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for lk_0, li_1, lj_1, lk_1 in T.grid(8, 2, 2, 8):
with T.block("S_gemm_update"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1)
j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1)
k = T.axis.reduce(64, lk_0 * 8 + lk_1)
T.reads(S_local[i, j], Q_smem[i, k], K_smem[j, k])
T.writes(S_local[i, j])
S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k]) * T.Cast("float32", K_smem[j, k]) * attn_score_scaling_factor * T.float32(0.18033688011112042)
T.tvm_storage_sync("shared")
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for li_1, lj_1 in T.grid(2, 2):
with T.block("S_store"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1)
j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1)
T.reads(S_local[i, j])
T.writes(S_smem[i, j])
S_smem[i, j] = S_local[i, j]
T.tvm_storage_sync("shared")
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
if row < 32:
with T.block("update1"):
T.reads(m_smem[row], kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:16], d_smem[row], m_prev[i])
T.writes(m_prev[i], m_new[i], d_new[i])
m_prev[i] = m_smem[row]
m_new[i] = m_smem[row]
row_: T.int32 = LH_start + row
for j in range(16):
if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]):
m_new[i] = T.max(m_new[i], S_smem[row, j])
d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i])
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
with T.block("update"):
T.reads(kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:16], m_new[i])
T.writes(S_smem[row, 0:16])
for j in range(16):
if row < 32:
row_: T.int32 = LH_start + row
if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]):
S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i])
else:
S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i])
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
if row < 32:
with T.block("update"):
T.reads(d_new[i], S_smem[row, 0:16], m_new[i], m_prev[i])
T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row])
for j in range(16):
d_new[i] = d_new[i] + S_smem[row, j]
m_smem[row] = m_new[i]
d_smem[row] = d_new[i]
m_prev_smem[row] = m_prev[i]
T.tvm_storage_sync("shared")
with T.block(""):
T.reads(m_prev_smem[0:32], m_smem[0:32], S_smem[0:32, 0:16], V_smem[0:16, 0:64])
T.writes(O_local[0:32, 0:64])
for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"):
for li_1_init, lj_1_init in T.grid(4, 4):
with T.block("O_gemm_init"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 4 + li_1_init)
j = T.axis.spatial(64, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 4 + lj_1_init)
T.reads()
T.writes(O_local[i, j])
O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i])
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for lk_0, lk_1, li_1, lj_1 in T.grid(2, 8, 4, 4):
with T.block("O_gemm_update"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1)
j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1)
k = T.axis.reduce(16, lk_0 * 8 + lk_1)
T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k], V_smem[k, j])
T.writes(O_local[i, j])
O_local[i, j] = O_local[i, j] + S_smem[i, k] * T.Cast("float32", V_smem[k, j])
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for li_1, lj_1 in T.grid(4, 4):
with T.block("O_store"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1)
j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1)
T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i])
T.writes(output[q_indptr[b_idx] + (LH_start + i), by, j])
cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i)
cur_H_qo: T.int32 = by
if cur_L < q_indptr[b_idx + 1]:
output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i])
for li_0 in range(1):
for li_1 in T.thread_binding(4, thread="threadIdx.y"):
for li_2 in T.thread_binding(32, thread="threadIdx.x"):
with T.block("lse_store"):
i = T.axis.spatial(32, li_0 * 128 + li_1 * 32 + li_2)
T.where((li_0 * 4 + li_1) * 32 + li_2 < 32)
T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i])
T.writes(lse[q_indptr[b_idx] + (LH_start + i), by])
cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i)
cur_H_qo: T.int32 = by
if cur_L < q_indptr[b_idx + 1]:
lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i])
tile_id[0] = tile_id[0] + 16
@T.prim_func
def batch_prefill_paged_kv_sliding_window(_0: T.int32, var_q: T.handle, var_q_indptr: T.handle, var_pages: T.handle, var_page_indptr: T.handle, var_page_values: T.handle, var_length_info: T.handle, var_k_rope_pos_offset: T.handle, var_q_rope_position: T.handle, var_output: T.handle, var_lse: T.handle, causal: T.int32, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32):
T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1})
total_len = T.int32(is_size_var=True)
q = T.match_buffer(var_q, (total_len, 8, 64), "float16")
batch_size = T.int32(is_size_var=True)
q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1)
max_num_pages = T.int32(is_size_var=True)
pages = T.match_buffer(var_pages, (max_num_pages, 2, 8, 16, 64), "float16", offset_factor=1)
page_indptr = T.match_buffer(var_page_indptr, (batch_size + 1,), "int32", offset_factor=1)
nnz_pages = T.int32(is_size_var=True)
page_values = T.match_buffer(var_page_values, (nnz_pages,), "int32", offset_factor=1)
length_info = T.match_buffer(var_length_info, (3, batch_size), "int32", offset_factor=1)
k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1)
q_rope_position = T.match_buffer(var_q_rope_position, (total_len,), "int32", offset_factor=1)
output = T.match_buffer(var_output, (total_len, 8, 64), "float16")
lse = T.match_buffer(var_lse, (total_len, 8))
# with T.block("root"):
for lbx in T.thread_binding(16, thread="blockIdx.x"):
for lby in T.thread_binding(8, thread="blockIdx.y"):
for lty in T.thread_binding(4, thread="threadIdx.y"):
for ltx in T.thread_binding(32, thread="threadIdx.x"):
with T.block("attn"):
bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx])
T.reads()
T.writes()
tile_id = T.alloc_buffer((1,), "int32", scope="local")
batch_idx = T.alloc_buffer((1,), "int32", scope="local")
batch_tiles = T.alloc_buffer((1,), "int32", scope="local")
batch_rows = T.alloc_buffer((1,), "int32", scope="local")
iterator = T.alloc_buffer((1,), "int32", scope="local")
kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local")
Q_smem = T.alloc_buffer((32, 64), "float16", scope="shared")
K_smem = T.alloc_buffer((16, 64), "float16", scope="shared")
V_smem = T.alloc_buffer((16, 64), "float16", scope="shared")
S_smem = T.alloc_buffer((32, 16), scope="shared")
S_local = T.alloc_buffer((32, 16), scope="local")
O_local = T.alloc_buffer((32, 64), scope="local")
m_smem = T.alloc_buffer((32,), scope="shared")
m_prev_smem = T.alloc_buffer((32,), scope="shared")
d_smem = T.alloc_buffer((32,), scope="shared")
m_new = T.alloc_buffer((1,), scope="local")
m_prev = T.alloc_buffer((1,), scope="local")
d_new = T.alloc_buffer((1,), scope="local")
tile_id[0] = bx
batch_idx[0] = 0
batch_rows[0] = q_indptr[1] - q_indptr[0]
batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32
while T.tvm_thread_invariant(batch_idx[0] < batch_size):
while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size:
tile_id[0] = tile_id[0] - batch_tiles[0]
batch_idx[0] = batch_idx[0] + 1
if batch_idx[0] < batch_size:
b_idx: T.int32 = batch_idx[0]
batch_rows[0] = q_indptr[b_idx + 1] - q_indptr[b_idx]
batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32
if T.tvm_thread_invariant(batch_idx[0] < batch_size):
b_idx: T.int32 = batch_idx[0]
LH_start: T.int32 = tile_id[0] * 32
q_indptr_val: T.int32 = q_indptr[b_idx]
cur_page_indptr_begin: T.int32 = page_indptr[b_idx]
cur_page_indptr_end: T.int32 = page_indptr[b_idx + 1]
kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 16 + length_info[0, b_idx] - length_info[1, b_idx] + length_info[2, b_idx], 0)
T.tvm_storage_sync("shared")
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
if row < 32:
m_smem[row] = T.float32(-50000.0)
d_smem[row] = T.float32(1.0)
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for li_1, lj_1 in T.grid(4, 4):
with T.block("O_init"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1)
j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1)
T.reads()
T.writes(O_local[i, j])
O_local[i, j] = T.float32(0.0)
T.tvm_storage_sync("shared")
for li_lj_fused_0 in range(4):
for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"):
for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"):
for li_lj_fused_3 in T.vectorized(4):
with T.block("Q_load"):
i = T.axis.spatial(32, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 64)
j = T.axis.spatial(64, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 64)
T.reads()
T.writes()
cur_L: T.int32 = q_indptr_val + (LH_start + i)
cur_H_qo: T.int32 = by
if cur_L < q_indptr[b_idx + 1]:
freq = T.float32()
Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, q[cur_L, cur_H_qo, j + 32] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 32]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), q[cur_L, cur_H_qo, j])
else:
Q_smem[i, j] = T.float16(0.0)
T.tvm_storage_sync("shared")
for iterator_1 in range((kv_chunk_len[0] + 15) // 16):
L_kv_start: T.int32 = iterator_1 * 16
for lz_ly_fused_0 in range(2):
for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"):
for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"):
for lz_ly_fused_3 in T.vectorized(4):
with T.block("K_load"):
i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64)
j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64)
T.reads()
T.writes()
cur_L: T.int32 = L_kv_start + i
if cur_L < kv_chunk_len[0]:
seq_offset: T.int32 = T.if_then_else(cur_L < length_info[2, b_idx], cur_L, cur_L - length_info[2, b_idx] + length_info[1, b_idx])
page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16]
page_offset: T.int32 = seq_offset % 16
freq = T.float32()
K_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", pages[page_no, 0, by, page_offset, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, pages[page_no, 0, by, page_offset, j + 32] * T.float16(-1.0), pages[page_no, 0, by, page_offset, j - 32]))), where={freq: T.Cast("float32", k_rope_pos_offset[b_idx] + cur_L) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), pages[page_no, 0, by, page_offset, j])
else:
K_smem[i, j] = T.float16(0.0)
T.tvm_storage_sync("shared")
for lz_ly_fused_0 in range(2):
for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"):
for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"):
for lz_ly_fused_3 in T.vectorized(4):
with T.block("V_load"):
i = T.axis.spatial(16, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64)
j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64)
T.reads()
T.writes()
cur_L: T.int32 = L_kv_start + i
if cur_L < kv_chunk_len[0]:
seq_offset: T.int32 = T.if_then_else(cur_L < length_info[2, b_idx], cur_L, cur_L - length_info[2, b_idx] + length_info[1, b_idx])
page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 16]
page_offset: T.int32 = seq_offset % 16
V_smem[i, j] = pages[page_no, 1, by, page_offset, j]
else:
V_smem[i, j] = T.float16(0.0)
T.tvm_storage_sync("shared")
with T.block(""):
T.reads(Q_smem[0:32, 0:64], K_smem[0:16, 0:64])
T.writes(S_local[0:32, 0:16])
for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"):
for li_1_init, lj_1_init in T.grid(2, 2):
with T.block("S_gemm_init"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 * 2 + li_1_init)
j = T.axis.spatial(16, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 2 + lj_1_init)
T.reads()
T.writes(S_local[i, j])
S_local[i, j] = T.float32(0.0)
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for lk_0, li_1, lj_1, lk_1 in T.grid(8, 2, 2, 8):
with T.block("S_gemm_update"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1)
j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1)
k = T.axis.reduce(64, lk_0 * 8 + lk_1)
T.reads(S_local[i, j], Q_smem[i, k], K_smem[j, k])
T.writes(S_local[i, j])
S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k]) * T.Cast("float32", K_smem[j, k]) * attn_score_scaling_factor * T.float32(0.18033688011112042)
T.tvm_storage_sync("shared")
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for li_1, lj_1 in T.grid(2, 2):
with T.block("S_store"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1)
j = T.axis.spatial(16, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 2 + lj_1)
T.reads(S_local[i, j])
T.writes(S_smem[i, j])
S_smem[i, j] = S_local[i, j]
T.tvm_storage_sync("shared")
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
if row < 32:
with T.block("update1"):
T.reads(m_smem[row], kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:16], d_smem[row], m_prev[i])
T.writes(m_prev[i], m_new[i], d_new[i])
m_prev[i] = m_smem[row]
m_new[i] = m_smem[row]
row_: T.int32 = LH_start + row
for j in range(16):
if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]):
m_new[i] = T.max(m_new[i], S_smem[row, j])
d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i])
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
with T.block("update"):
T.reads(kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:16], m_new[i])
T.writes(S_smem[row, 0:16])
for j in range(16):
if row < 32:
row_: T.int32 = LH_start + row
if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]):
S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i])
else:
S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i])
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
if row < 32:
with T.block("update"):
T.reads(d_new[i], S_smem[row, 0:16], m_new[i], m_prev[i])
T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row])
for j in range(16):
d_new[i] = d_new[i] + S_smem[row, j]
m_smem[row] = m_new[i]
d_smem[row] = d_new[i]
m_prev_smem[row] = m_prev[i]
T.tvm_storage_sync("shared")
with T.block(""):
T.reads(m_prev_smem[0:32], m_smem[0:32], S_smem[0:32, 0:16], V_smem[0:16, 0:64])
T.writes(O_local[0:32, 0:64])
for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"):
for li_1_init, lj_1_init in T.grid(4, 4):
with T.block("O_gemm_init"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 4 + li_1_init)
j = T.axis.spatial(64, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 4 + lj_1_init)
T.reads()
T.writes(O_local[i, j])
O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i])
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for lk_0, lk_1, li_1, lj_1 in T.grid(2, 8, 4, 4):
with T.block("O_gemm_update"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1)
j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1)
k = T.axis.reduce(16, lk_0 * 8 + lk_1)
T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k], V_smem[k, j])
T.writes(O_local[i, j])
O_local[i, j] = O_local[i, j] + S_smem[i, k] * T.Cast("float32", V_smem[k, j])
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for li_1, lj_1 in T.grid(4, 4):
with T.block("O_store"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1)
j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1)
T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i])
T.writes(output[q_indptr[b_idx] + (LH_start + i), by, j])
cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i)
cur_H_qo: T.int32 = by
if cur_L < q_indptr[b_idx + 1]:
output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i])
for li_0 in range(1):
for li_1 in T.thread_binding(4, thread="threadIdx.y"):
for li_2 in T.thread_binding(32, thread="threadIdx.x"):
with T.block("lse_store"):
i = T.axis.spatial(32, li_0 * 128 + li_1 * 32 + li_2)
T.where((li_0 * 4 + li_1) * 32 + li_2 < 32)
T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i])
T.writes(lse[q_indptr[b_idx] + (LH_start + i), by])
cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i)
cur_H_qo: T.int32 = by
if cur_L < q_indptr[b_idx + 1]:
lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i])
tile_id[0] = tile_id[0] + 16
@T.prim_func
def batch_prefill_ragged_kv(var_q: T.handle, var_q_indptr: T.handle, var_k: T.handle, var_v: T.handle, var_kv_indptr: T.handle, var_q_rope_position: T.handle, var_k_rope_pos_offset: T.handle, var_output: T.handle, var_lse: T.handle, causal: T.int32, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32):
T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1})
qo_len = T.int32(is_size_var=True)
q = T.match_buffer(var_q, (qo_len, 8, 64), "float16")
batch_size = T.int32(is_size_var=True)
q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1)
kv_len = T.int32(is_size_var=True)
k = T.match_buffer(var_k, (kv_len, 8, 64), "float16")
v = T.match_buffer(var_v, (kv_len, 8, 64), "float16")
kv_indptr = T.match_buffer(var_kv_indptr, (batch_size + 1,), "int32", offset_factor=1)
q_rope_position = T.match_buffer(var_q_rope_position, (qo_len,), "int32", offset_factor=1)
k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1)
output = T.match_buffer(var_output, (qo_len, 8, 64), "float16")
lse = T.match_buffer(var_lse, (qo_len, 8))
# with T.block("root"):
for lbx in T.thread_binding(8, thread="blockIdx.x"):
for lby in T.thread_binding(8, thread="blockIdx.y"):
for lty in T.thread_binding(4, thread="threadIdx.y"):
for ltx in T.thread_binding(32, thread="threadIdx.x"):
with T.block("attn"):
bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx])
T.reads()
T.writes()
tile_id = T.alloc_buffer((1,), "int32", scope="local")
batch_idx = T.alloc_buffer((1,), "int32", scope="local")
batch_tiles = T.alloc_buffer((1,), "int32", scope="local")
batch_rows = T.alloc_buffer((1,), "int32", scope="local")
iterator = T.alloc_buffer((1,), "int32", scope="local")
kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local")
Q_smem = T.alloc_buffer((32, 64), "float16", scope="shared")
K_smem = T.alloc_buffer((64, 32), "float16", scope="shared")
V_smem = T.alloc_buffer((32, 64), "float16", scope="shared")
S_smem = T.alloc_buffer((32, 32), scope="shared")
S_local = T.alloc_buffer((32, 32), scope="local")
O_local = T.alloc_buffer((32, 64), scope="local")
m_smem = T.alloc_buffer((32,), scope="shared")
m_prev_smem = T.alloc_buffer((32,), scope="shared")
d_smem = T.alloc_buffer((32,), scope="shared")
m_new = T.alloc_buffer((1,), scope="local")
m_prev = T.alloc_buffer((1,), scope="local")
d_new = T.alloc_buffer((1,), scope="local")
tile_id[0] = bx
batch_idx[0] = 0
batch_rows[0] = q_indptr[1] - q_indptr[0]
batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32
while T.tvm_thread_invariant(batch_idx[0] < batch_size):
while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size:
tile_id[0] = tile_id[0] - batch_tiles[0]
batch_idx[0] = batch_idx[0] + 1
if batch_idx[0] < batch_size:
b_idx: T.int32 = batch_idx[0]
batch_rows[0] = q_indptr[b_idx + 1] - q_indptr[b_idx]
batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32
if T.tvm_thread_invariant(batch_idx[0] < batch_size):
b_idx: T.int32 = batch_idx[0]
q_indptr_val: T.int32 = q_indptr[b_idx]
LH_start: T.int32 = tile_id[0] * 32
kv_chunk_len[0] = kv_indptr[b_idx + 1] - kv_indptr[b_idx]
T.tvm_storage_sync("shared")
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
if row < 32:
m_smem[row] = T.float32(-50000.0)
d_smem[row] = T.float32(1.0)
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for li_1 in range(4):
for lj_1_0 in T.unroll(1):
for lj_1_1 in T.vectorized(4):
with T.block("O_init"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1)
j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1_0 * 4 + lj_1_1)
T.reads()
T.writes(O_local[i, j])
O_local[i, j] = T.float32(0.0)
T.tvm_storage_sync("shared")
for li_1_lj_0_1_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_1_lj_0_1_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for li_0, lj_0_0 in T.grid(1, 2):
for lj_1 in T.vectorized(8):
with T.block("Q_load"):
i = T.axis.spatial(32, li_0 * 32 + (li_1_lj_0_1_fused_0 * 32 + li_1_lj_0_1_fused_1) // 4)
j = T.axis.spatial(64, lj_0_0 * 32 + (li_1_lj_0_1_fused_0 * 32 + li_1_lj_0_1_fused_1) % 4 * 8 + lj_1)
T.reads()
T.writes()
cur_L: T.int32 = q_indptr_val + (LH_start + i)
cur_H_qo: T.int32 = by
if cur_L < q_indptr[b_idx + 1]:
freq = T.float32()
Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, q[cur_L, cur_H_qo, j + 32] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 32]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), q[cur_L, cur_H_qo, j])
else:
Q_smem[i, j] = T.float16(0.0)
T.tvm_storage_sync("shared")
for iterator_1 in range((kv_chunk_len[0] + 31) // 32):
L_kv_start: T.int32 = iterator_1 * 32
L_kv_base: T.int32 = kv_indptr[b_idx]
for lz_1_ly_0_1_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for lz_1_ly_0_1_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for lz_0, ly_0_0 in T.grid(1, 2):
for ly_1 in T.vectorized(8):
with T.block("K_load"):
i = T.axis.spatial(32, lz_0 * 32 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) // 4)
j = T.axis.spatial(64, ly_0_0 * 32 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) % 4 * 8 + ly_1)
T.reads(kv_chunk_len[0], k_rope_pos_offset[b_idx], k[L_kv_base + L_kv_start + i, by, j - 32:j - 32 + 65])
T.writes(K_smem[j, i])
cur_L: T.int32 = L_kv_start + i
if cur_L < kv_chunk_len[0]:
freq = T.float32()
K_smem[j, i] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", k[L_kv_base + cur_L, by, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, k[L_kv_base + cur_L, by, j + 32] * T.float16(-1.0), k[L_kv_base + cur_L, by, j - 32]))), where={freq: T.Cast("float32", k_rope_pos_offset[b_idx] + cur_L) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), k[L_kv_base + cur_L, by, j])
else:
K_smem[j, i] = T.float16(0.0)
T.tvm_storage_sync("shared")
for lz_1_ly_0_1_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for lz_1_ly_0_1_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for lz_0, ly_0_0 in T.grid(1, 2):
for ly_1 in T.vectorized(8):
with T.block("V_load"):
i = T.axis.spatial(32, lz_0 * 32 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) // 4)
j = T.axis.spatial(64, ly_0_0 * 32 + (lz_1_ly_0_1_fused_0 * 32 + lz_1_ly_0_1_fused_1) % 4 * 8 + ly_1)
T.reads()
T.writes()
cur_L: T.int32 = L_kv_start + i
if cur_L < kv_chunk_len[0]:
V_smem[i, j] = v[L_kv_base + cur_L, by, j]
else:
V_smem[i, j] = T.float16(0.0)
T.tvm_storage_sync("shared")
with T.block(""):
T.reads(Q_smem[0:32, 0:64], K_smem[0:64, 0:32])
T.writes(S_local[0:32, 0:32])
for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"):
for li_1_init in T.unroll(2):
for lj_1_0_init in T.unroll(1):
for lj_1_1_init in T.vectorized(4):
with T.block("S_gemm_init"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 * 2 + li_1_init)
j = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 4 + lj_1_0_init * 4 + lj_1_1_init)
T.reads()
T.writes(S_local[i, j])
S_local[i, j] = T.float32(0.0)
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for lk_0 in range(4):
for li_1 in T.unroll(2):
for lj_1_0 in T.unroll(1):
for lj_1_1 in T.vectorized(4):
for lk_1 in range(16):
with T.block("S_gemm_update"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1)
j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1_0 * 4 + lj_1_1)
k_1 = T.axis.reduce(64, lk_0 * 16 + lk_1)
T.reads(S_local[i, j], Q_smem[i, k_1], K_smem[k_1, j])
T.writes(S_local[i, j])
S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k_1]) * T.Cast("float32", K_smem[k_1, j]) * attn_score_scaling_factor * T.float32(0.18033688011112042)
T.tvm_storage_sync("shared")
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for li_1 in range(2):
for lj_1_0 in T.unroll(1):
for lj_1_1 in T.vectorized(4):
with T.block("S_store"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1)
j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1_0 * 4 + lj_1_1)
T.reads(S_local[i, j])
T.writes(S_smem[i, j])
S_smem[i, j] = S_local[i, j]
T.tvm_storage_sync("shared")
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
if row < 32:
with T.block("update1"):
T.reads(m_smem[row], kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:32], d_smem[row], m_prev[i])
T.writes(m_prev[i], m_new[i], d_new[i])
m_prev[i] = m_smem[row]
m_new[i] = m_smem[row]
row_: T.int32 = LH_start + row
for j in range(32):
if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]):
m_new[i] = T.max(m_new[i], S_smem[row, j])
d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i])
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
with T.block("update"):
T.reads(kv_chunk_len[0], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:32], m_new[i])
T.writes(S_smem[row, 0:32])
for j in range(32):
if row < 32:
row_: T.int32 = LH_start + row
if T.if_then_else(causal > 0, L_kv_start + j < kv_chunk_len[0] - (q_indptr[b_idx + 1] - q_indptr[b_idx]) + row_ + 1, L_kv_start + j < kv_chunk_len[0]):
S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i])
else:
S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i])
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
if row < 32:
with T.block("update"):
T.reads(d_new[i], S_smem[row, 0:32], m_new[i], m_prev[i])
T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row])
for j in range(32):
d_new[i] = d_new[i] + S_smem[row, j]
m_smem[row] = m_new[i]
d_smem[row] = d_new[i]
m_prev_smem[row] = m_prev[i]
T.tvm_storage_sync("shared")
with T.block(""):
T.reads(m_prev_smem[0:32], m_smem[0:32], S_smem[0:32, 0:32], V_smem[0:32, 0:64])
T.writes(O_local[0:32, 0:64])
for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"):
for li_1_init in T.unroll(4):
for lj_1_0_init in T.unroll(1):
for lj_1_1_init in T.vectorized(4):
with T.block("O_gemm_init"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 4 + li_1_init)
j = T.axis.spatial(64, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 4 + lj_1_0_init * 4 + lj_1_1_init)
T.reads()
T.writes(O_local[i, j])
O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i])
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for lk_0, lk_1 in T.grid(2, 16):
for li_1 in T.unroll(4):
for lj_1_0 in T.unroll(1):
for lj_1_1 in T.vectorized(4):
with T.block("O_gemm_update"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1)
j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1_0 * 4 + lj_1_1)
k_1 = T.axis.reduce(32, lk_0 * 16 + lk_1)
T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k_1], V_smem[k_1, j])
T.writes(O_local[i, j])
O_local[i, j] = O_local[i, j] + S_smem[i, k_1] * T.Cast("float32", V_smem[k_1, j])
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for li_1 in range(4):
for lj_1_0 in T.unroll(1):
for lj_1_1 in T.vectorized(4):
with T.block("O_store"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1)
j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1_0 * 4 + lj_1_1)
T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i])
T.writes(output[q_indptr[b_idx] + (LH_start + i), by, j])
cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i)
cur_H_qo: T.int32 = by
if cur_L < q_indptr[b_idx + 1]:
output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i])
for li_0 in range(1):
for li_1 in T.thread_binding(4, thread="threadIdx.y"):
for li_2 in T.thread_binding(32, thread="threadIdx.x"):
with T.block("lse_store"):
i = T.axis.spatial(32, li_0 * 128 + li_1 * 32 + li_2)
T.where((li_0 * 4 + li_1) * 32 + li_2 < 32)
T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i])
T.writes(lse[q_indptr[b_idx] + (LH_start + i), by])
cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i)
cur_H_qo: T.int32 = by
if cur_L < q_indptr[b_idx + 1]:
lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i])
tile_id[0] = tile_id[0] + 8
@T.prim_func
def batch_tree_attn(var_q: T.handle, var_q_indptr: T.handle, var_k: T.handle, var_v: T.handle, var_kv_indptr: T.handle, var_q_rope_position: T.handle, var_mn_indptr: T.handle, var_mask: T.handle, var_output: T.handle, var_lse: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32, batch_size: T.int32):
T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1})
qo_len = T.int32(is_size_var=True)
q = T.match_buffer(var_q, (qo_len, 8, 64), "float16")
q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1)
kv_len = T.int32(is_size_var=True)
k = T.match_buffer(var_k, (kv_len, 8, 64), "float16")
v = T.match_buffer(var_v, (kv_len, 8, 64), "float16")
kv_indptr = T.match_buffer(var_kv_indptr, (batch_size + 1,), "int32", offset_factor=1)
q_rope_position = T.match_buffer(var_q_rope_position, (qo_len,), "int32", offset_factor=1)
mn_indptr = T.match_buffer(var_mn_indptr, (batch_size + 1,), "int32", offset_factor=1)
tree_size = T.int32(is_size_var=True)
mask = T.match_buffer(var_mask, (tree_size, 2), "int32", offset_factor=1)
output = T.match_buffer(var_output, (qo_len, 8, 64), "float16")
lse = T.match_buffer(var_lse, (qo_len, 8))
# with T.block("root"):
for lbx in T.thread_binding(16, thread="blockIdx.x"):
for lby in T.thread_binding(8, thread="blockIdx.y"):
for lty in T.thread_binding(4, thread="threadIdx.y"):
for ltx in T.thread_binding(32, thread="threadIdx.x"):
with T.block("attn"):
bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx])
T.reads()
T.writes()
tile_id = T.alloc_buffer((1,), "int32", scope="local")
batch_idx = T.alloc_buffer((1,), "int32", scope="local")
batch_tiles = T.alloc_buffer((1,), "int32", scope="local")
batch_rows = T.alloc_buffer((1,), "int32", scope="local")
iterator = T.alloc_buffer((1,), "int32", scope="local")
kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local")
Q_smem = T.alloc_buffer((32, 64), "float16", scope="shared")
K_smem = T.alloc_buffer((32, 64), "float16", scope="shared")
V_smem = T.alloc_buffer((32, 64), "float16", scope="shared")
S_smem = T.alloc_buffer((32, 32), scope="shared")
S_local = T.alloc_buffer((32, 32), scope="local")
O_local = T.alloc_buffer((32, 64), scope="local")
m_smem = T.alloc_buffer((32,), scope="shared")
m_prev_smem = T.alloc_buffer((32,), scope="shared")
d_smem = T.alloc_buffer((32,), scope="shared")
m_new = T.alloc_buffer((1,), scope="local")
m_prev = T.alloc_buffer((1,), scope="local")
d_new = T.alloc_buffer((1,), scope="local")
tile_id[0] = bx
batch_idx[0] = 0
batch_rows[0] = q_indptr[1] - q_indptr[0]
batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32
while T.tvm_thread_invariant(batch_idx[0] < batch_size):
while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size:
tile_id[0] = tile_id[0] - batch_tiles[0]
batch_idx[0] = batch_idx[0] + 1
if batch_idx[0] < batch_size:
b_idx: T.int32 = batch_idx[0]
batch_rows[0] = q_indptr[b_idx + 1] - q_indptr[b_idx]
batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32
if T.tvm_thread_invariant(batch_idx[0] < batch_size):
b_idx: T.int32 = batch_idx[0]
LH_start: T.int32 = tile_id[0] * 32
q_indptr_val: T.int32 = q_indptr[b_idx]
kv_chunk_len[0] = kv_indptr[b_idx + 1] - kv_indptr[b_idx]
T.tvm_storage_sync("shared")
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
if row < 32:
m_smem[row] = T.float32(-50000.0)
d_smem[row] = T.float32(1.0)
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for li_1, lj_1 in T.grid(4, 4):
with T.block("O_init"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1)
j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1)
T.reads()
T.writes(O_local[i, j])
O_local[i, j] = T.float32(0.0)
T.tvm_storage_sync("shared")
for li_lj_fused_0 in range(4):
for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"):
for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"):
for li_lj_fused_3 in T.vectorized(4):
with T.block("Q_load"):
i = T.axis.spatial(32, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 64)
j = T.axis.spatial(64, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 64)
T.reads()
T.writes()
cur_L: T.int32 = q_indptr_val + (LH_start + i)
cur_H_qo: T.int32 = by
if cur_L < q_indptr[b_idx + 1]:
freq = T.float32()
Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, q[cur_L, cur_H_qo, j + 32] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 32]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), q[cur_L, cur_H_qo, j])
else:
Q_smem[i, j] = T.float16(0.0)
T.tvm_storage_sync("shared")
for iterator_1 in range((kv_chunk_len[0] + 31) // 32):
L_kv_start: T.int32 = iterator_1 * 32
L_kv_base: T.int32 = kv_indptr[b_idx]
for lz_ly_fused_0 in range(4):
for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"):
for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"):
for lz_ly_fused_3 in T.vectorized(4):
with T.block("KV_load"):
i = T.axis.spatial(32, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64)
j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64)
T.reads()
T.writes()
cur_L: T.int32 = L_kv_base + L_kv_start + i
if L_kv_start + i < kv_chunk_len[0]:
freq = T.float32()
K_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", k[cur_L, by, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, k[cur_L, by, j + 32] * T.float16(-1.0), k[cur_L, by, j - 32]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), k[cur_L, by, j])
V_smem[i, j] = v[cur_L, by, j]
else:
K_smem[i, j] = T.float16(0.0)
V_smem[i, j] = T.float16(0.0)
T.tvm_storage_sync("shared")
with T.block(""):
T.reads(Q_smem[0:32, 0:64], K_smem[0:32, 0:64])
T.writes(S_local[0:32, 0:32])
for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"):
for li_1_init, lj_1_init in T.grid(2, 4):
with T.block("S_gemm_init"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 * 2 + li_1_init)
j = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 4 + lj_1_init)
T.reads()
T.writes(S_local[i, j])
S_local[i, j] = T.float32(0.0)
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for lk_0, li_1, lj_1, lk_1 in T.grid(8, 2, 4, 8):
with T.block("S_gemm_update"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1)
j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1)
k_1 = T.axis.reduce(64, lk_0 * 8 + lk_1)
T.reads(S_local[i, j], Q_smem[i, k_1], K_smem[j, k_1])
T.writes(S_local[i, j])
S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k_1]) * T.Cast("float32", K_smem[j, k_1]) * attn_score_scaling_factor * T.float32(0.18033688011112042)
T.tvm_storage_sync("shared")
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for li_1, lj_1 in T.grid(2, 4):
with T.block("S_store"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1)
j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1)
T.reads(S_local[i, j])
T.writes(S_smem[i, j])
S_smem[i, j] = S_local[i, j]
T.tvm_storage_sync("shared")
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
if row < 32:
with T.block("update1"):
T.reads(m_smem[row], kv_chunk_len[0], mn_indptr[b_idx:b_idx + 2], mask[T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]):T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:32], d_smem[row], m_prev[i])
T.writes(m_prev[i], m_new[i], d_new[i])
m_prev[i] = m_smem[row]
m_new[i] = m_smem[row]
row_: T.int32 = LH_start + row
for j in range(32):
if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) or mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 0] and mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 1]):
m_new[i] = T.max(m_new[i], S_smem[row, j])
d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i])
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
with T.block("update"):
T.reads(kv_chunk_len[0], mn_indptr[b_idx:b_idx + 2], mask[T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]):T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min(LH_start + row + mn_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + mn_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:32], m_new[i])
T.writes(S_smem[row, 0:32])
for j in range(32):
if row < 32:
row_: T.int32 = LH_start + row
if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) or mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 0] and mask[mn_indptr[b_idx] + (row_ + (mn_indptr[b_idx + 1] - mn_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < mask[mn_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (mn_indptr[b_idx + 1] - mn_indptr[b_idx]))), 1]):
S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i])
else:
S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i])
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
if row < 32:
with T.block("update"):
T.reads(d_new[i], S_smem[row, 0:32], m_new[i], m_prev[i])
T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row])
for j in range(32):
d_new[i] = d_new[i] + S_smem[row, j]
m_smem[row] = m_new[i]
d_smem[row] = d_new[i]
m_prev_smem[row] = m_prev[i]
T.tvm_storage_sync("shared")
with T.block(""):
T.reads(m_prev_smem[0:32], m_smem[0:32], S_smem[0:32, 0:32], V_smem[0:32, 0:64])
T.writes(O_local[0:32, 0:64])
for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"):
for li_1_init, lj_1_init in T.grid(4, 4):
with T.block("O_gemm_init"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 4 + li_1_init)
j = T.axis.spatial(64, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 4 + lj_1_init)
T.reads()
T.writes(O_local[i, j])
O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i])
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for lk_0, lk_1, li_1, lj_1 in T.grid(4, 8, 4, 4):
with T.block("O_gemm_update"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1)
j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1)
k_1 = T.axis.reduce(32, lk_0 * 8 + lk_1)
T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k_1], V_smem[k_1, j])
T.writes(O_local[i, j])
O_local[i, j] = O_local[i, j] + S_smem[i, k_1] * T.Cast("float32", V_smem[k_1, j])
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for li_1, lj_1 in T.grid(4, 4):
with T.block("O_store"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1)
j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1)
T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i])
T.writes(output[q_indptr[b_idx] + (LH_start + i), by, j])
cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i)
cur_H_qo: T.int32 = by
if cur_L < q_indptr[b_idx + 1]:
output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i])
for li_0 in range(1):
for li_1 in T.thread_binding(4, thread="threadIdx.y"):
for li_2 in T.thread_binding(32, thread="threadIdx.x"):
with T.block("lse_store"):
i = T.axis.spatial(32, li_0 * 128 + li_1 * 32 + li_2)
T.where((li_0 * 4 + li_1) * 32 + li_2 < 32)
T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i])
T.writes(lse[q_indptr[b_idx] + (LH_start + i), by])
cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i)
cur_H_qo: T.int32 = by
if cur_L < q_indptr[b_idx + 1]:
lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i])
tile_id[0] = tile_id[0] + 16
@T.prim_func
def chunk_lse(var_A: T.handle, var_temperature: T.handle, var_chunked_sum: T.handle, var_chunked_max: T.handle):
T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)})
batch_size, vocab_size = T.int64(is_size_var=True), T.int64(is_size_var=True)
A = T.match_buffer(var_A, (batch_size, vocab_size))
temperature = T.match_buffer(var_temperature, (batch_size,))
num_chunks = T.int64(is_size_var=True)
chunked_sum = T.match_buffer(var_chunked_sum, (batch_size, num_chunks))
chunked_max = T.match_buffer(var_chunked_max, (batch_size, num_chunks))
# with T.block("root"):
A_pad = T.alloc_buffer((batch_size, num_chunks, T.int64(4096)))
temp_max = T.alloc_buffer((batch_size, num_chunks))
temp_sum = T.alloc_buffer((batch_size, num_chunks))
for l0, l1, l2 in T.grid(batch_size, num_chunks, T.int64(4096)):
with T.block("pad"):
v0, v1, v2 = T.axis.remap("SSS", [l0, l1, l2])
T.reads(temperature[v0], A[v0, v1 * T.int64(4096) + v2])
T.writes(A_pad[v0, v1, v2])
A_pad[v0, v1, v2] = T.if_then_else(v1 * T.int64(4096) + v2 < vocab_size, T.if_then_else(temperature[v0] > T.float32(1.0000000000000001e-05), A[v0, v1 * T.int64(4096) + v2] / temperature[v0], A[v0, v1 * T.int64(4096) + v2]), T.float32(-340282346638528859811704183484516925440.0))
for l0, l1, l2 in T.grid(batch_size, num_chunks, T.int64(4096)):
with T.block("max"):
v0, v1, v2 = T.axis.remap("SSR", [l0, l1, l2])
T.reads(A_pad[v0, v1, v2])
T.writes(temp_max[v0, v1])
with T.init():
temp_max[v0, v1] = T.float32(-340282346638528859811704183484516925440.0)
temp_max[v0, v1] = T.max(temp_max[v0, v1], A_pad[v0, v1, v2])
for l0, l1, l2 in T.grid(batch_size, num_chunks, T.int64(4096)):
with T.block("sum_exp"):
v0, v1, v2 = T.axis.remap("SSR", [l0, l1, l2])
T.reads(temperature[v0], A_pad[v0, v1, v2], temp_max[v0, v1])
T.writes(temp_sum[v0, v1])
with T.init():
temp_sum[v0, v1] = T.float32(0.0)
temp_sum[v0, v1] = temp_sum[v0, v1] + T.if_then_else(v1 * T.int64(4096) + v2 < vocab_size, T.Select(temperature[v0] > T.float32(1.0000000000000001e-05), T.exp(A_pad[v0, v1, v2] - temp_max[v0, v1]), T.Cast("float32", A_pad[v0, v1, v2] == temp_max[v0, v1])), T.float32(0.0))
for l0, l1, l2 in T.grid(batch_size, num_chunks, T.int64(1)):
with T.block("log"):
v0, v1, v2 = T.axis.remap("SSS", [l0, l1, l2])
T.reads(temperature[v0], temp_sum[v0, v1], temp_max[v0, v1])
T.writes(chunked_sum[v0, v1], chunked_max[v0, v1])
chunked_sum[v0, v1] = T.Select(temperature[v0] > T.float32(1.0000000000000001e-05), T.log(temp_sum[v0, v1]), temp_sum[v0, v1])
chunked_max[v0, v1] = temp_max[v0, v1]
@T.prim_func
def compact_kv_copy(var_pages: T.handle, var_copy_length_indptr: T.handle, var_copy_src_dst_pos: T.handle, batch_size: T.int32):
T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1})
num_pages = T.int32()
pages = T.match_buffer(var_pages, (num_pages, 2, 8, 16, 64), "float16", offset_factor=1)
copy_length_indptr = T.match_buffer(var_copy_length_indptr, (batch_size + 1,), "int32", offset_factor=1)
total_copy_length = T.int32()
copy_src_dst_pos = T.match_buffer(var_copy_src_dst_pos, (2, total_copy_length), "int32", offset_factor=1)
with T.block("root"):
T.reads()
T.writes()
for bhd_o in T.thread_binding(batch_size * 2, thread="blockIdx.x"):
for bhd_i in T.thread_binding(256, thread="threadIdx.x"):
b: T.int32 = (bhd_o * 256 + bhd_i) // 512
h: T.int32 = (bhd_o * 256 + bhd_i) // 64 % 8
d: T.int32 = (bhd_o * 256 + bhd_i) % 64
if bhd_o * 256 + bhd_i < batch_size * 8 * 64:
for i in range(copy_length_indptr[b + 1] - copy_length_indptr[b]):
src_pos: T.int32 = copy_src_dst_pos[0, copy_length_indptr[b] + i]
dst_pos: T.int32 = copy_src_dst_pos[1, copy_length_indptr[b] + i]
pages[dst_pos // 16, 0, h, dst_pos % 16, d] = pages[src_pos // 16, 0, h, src_pos % 16, d]
pages[dst_pos // 16, 1, h, dst_pos % 16, d] = pages[src_pos // 16, 1, h, src_pos % 16, d]
@T.prim_func
def copy_single_page(var_pages: T.handle, src_page_id: T.int64, tgt_page_id: T.int64, copy_length: T.int64):
T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1})
num_pages, page_size = T.int32(), T.int64()
pages = T.match_buffer(var_pages, (num_pages, 2, 8, page_size, 64), "float16", offset_factor=1)
# with T.block("root"):
for b in T.thread_binding(copy_length * T.int64(2), thread="blockIdx.x"):
for t in T.thread_binding(256, thread="threadIdx.x"):
with T.block("copy"):
vh = T.axis.spatial(8, T.Cast("int32", (b * T.int64(256) + T.Cast("int64", t)) // (copy_length * T.int64(64))))
vp = T.axis.spatial(copy_length, (b * T.int64(256) + T.Cast("int64", t)) % (copy_length * T.int64(64)) // T.int64(64))
vd = T.axis.spatial(64, T.Cast("int32", (b * T.int64(256) + T.Cast("int64", t)) % T.int64(64)))
T.where(b * T.int64(256) + T.Cast("int64", t) < copy_length * T.int64(8) * T.int64(64))
T.reads(pages[src_page_id, 0:2, vh, vp, vd])
T.writes(pages[tgt_page_id, 0:2, vh, vp, vd])
pages[tgt_page_id, 0, vh, vp, vd] = pages[src_page_id, 0, vh, vp, vd]
pages[tgt_page_id, 1, vh, vp, vd] = pages[src_page_id, 1, vh, vp, vd]
@T.prim_func(private=True)
def dequantize(var_gpt_neox_embed_in_q_weight: T.handle, var_gpt_neox_embed_in_q_scale: T.handle, var_dequantize: T.handle):
T.func_attr({"target": T.target({"keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)})
vocab_size = T.int64()
gpt_neox_embed_in_q_weight = T.match_buffer(var_gpt_neox_embed_in_q_weight, (vocab_size, T.int64(64)), "uint32")
gpt_neox_embed_in_q_scale = T.match_buffer(var_gpt_neox_embed_in_q_scale, (vocab_size, T.int64(16)), "float16")
dequantize = T.match_buffer(var_dequantize, (vocab_size, T.int64(512)), "float16")
# with T.block("root"):
compute = T.alloc_buffer((vocab_size, T.int64(512)), "float16")
for i0, i1 in T.grid(vocab_size, T.int64(512)):
with T.block("compute"):
v_i0, v_i1 = T.axis.remap("SS", [i0, i1])
T.reads(gpt_neox_embed_in_q_weight[v_i0, v_i1 // T.int64(8)])
T.writes(compute[v_i0, v_i1])
compute[v_i0, v_i1] = T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_embed_in_q_weight[v_i0, v_i1 // T.int64(8)], T.Cast("uint32", v_i1 % T.int64(8) * T.int64(4))), T.uint32(15)))
for i0, i1 in T.grid(vocab_size, T.int64(512)):
with T.block("dequantize"):
v_i0, v_i1 = T.axis.remap("SS", [i0, i1])
T.reads(compute[v_i0, v_i1], gpt_neox_embed_in_q_scale[v_i0, v_i1 // T.int64(32)])
T.writes(dequantize[v_i0, v_i1])
dequantize[v_i0, v_i1] = (compute[v_i0, v_i1] - T.float16(7.0)) * gpt_neox_embed_in_q_scale[v_i0, v_i1 // T.int64(32)]
@T.prim_func(private=True)
def dequantize1(gpt_neox_layers_0_attention_query_key_value_q_weight1: T.Buffer((T.int64(1536), T.int64(64)), "uint32"), gpt_neox_layers_0_attention_query_key_value_q_scale1: T.Buffer((T.int64(1536), T.int64(16)), "float16"), dequantize: T.Buffer((T.int64(1536), T.int64(512)), "float16")):
T.func_attr({"target": T.target({"keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)})
# with T.block("root"):
compute = T.alloc_buffer((T.int64(1536), T.int64(512)), "float16")
for i0, i1 in T.grid(T.int64(1536), T.int64(512)):
with T.block("compute"):
v_i0, v_i1 = T.axis.remap("SS", [i0, i1])
T.reads(gpt_neox_layers_0_attention_query_key_value_q_weight1[v_i0, v_i1 // T.int64(8)])
T.writes(compute[v_i0, v_i1])
compute[v_i0, v_i1] = T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_attention_query_key_value_q_weight1[v_i0, v_i1 // T.int64(8)], T.Cast("uint32", v_i1 % T.int64(8) * T.int64(4))), T.uint32(15)))
for i0, i1 in T.grid(T.int64(1536), T.int64(512)):
with T.block("dequantize"):
v_i0, v_i1 = T.axis.remap("SS", [i0, i1])
T.reads(compute[v_i0, v_i1], gpt_neox_layers_0_attention_query_key_value_q_scale1[v_i0, v_i1 // T.int64(32)])
T.writes(dequantize[v_i0, v_i1])
dequantize[v_i0, v_i1] = (compute[v_i0, v_i1] - T.float16(7.0)) * gpt_neox_layers_0_attention_query_key_value_q_scale1[v_i0, v_i1 // T.int64(32)]
@T.prim_func(private=True)
def dequantize2(gpt_neox_layers_0_attention_dense_q_weight1: T.Buffer((T.int64(512), T.int64(64)), "uint32"), gpt_neox_layers_0_attention_dense_q_scale1: T.Buffer((T.int64(512), T.int64(16)), "float16"), dequantize: T.Buffer((T.int64(512), T.int64(512)), "float16")):
T.func_attr({"target": T.target({"keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)})
# with T.block("root"):
compute = T.alloc_buffer((T.int64(512), T.int64(512)), "float16")
for i0, i1 in T.grid(T.int64(512), T.int64(512)):
with T.block("compute"):
v_i0, v_i1 = T.axis.remap("SS", [i0, i1])
T.reads(gpt_neox_layers_0_attention_dense_q_weight1[v_i0, v_i1 // T.int64(8)])
T.writes(compute[v_i0, v_i1])
compute[v_i0, v_i1] = T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_attention_dense_q_weight1[v_i0, v_i1 // T.int64(8)], T.Cast("uint32", v_i1 % T.int64(8) * T.int64(4))), T.uint32(15)))
for i0, i1 in T.grid(T.int64(512), T.int64(512)):
with T.block("dequantize"):
v_i0, v_i1 = T.axis.remap("SS", [i0, i1])
T.reads(compute[v_i0, v_i1], gpt_neox_layers_0_attention_dense_q_scale1[v_i0, v_i1 // T.int64(32)])
T.writes(dequantize[v_i0, v_i1])
dequantize[v_i0, v_i1] = (compute[v_i0, v_i1] - T.float16(7.0)) * gpt_neox_layers_0_attention_dense_q_scale1[v_i0, v_i1 // T.int64(32)]
@T.prim_func(private=True)
def dequantize3(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight1: T.Buffer((T.int64(2048), T.int64(64)), "uint32"), gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale1: T.Buffer((T.int64(2048), T.int64(16)), "float16"), dequantize: T.Buffer((T.int64(2048), T.int64(512)), "float16")):
T.func_attr({"target": T.target({"keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)})
# with T.block("root"):
compute = T.alloc_buffer((T.int64(2048), T.int64(512)), "float16")
for i0, i1 in T.grid(T.int64(2048), T.int64(512)):
with T.block("compute"):
v_i0, v_i1 = T.axis.remap("SS", [i0, i1])
T.reads(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight1[v_i0, v_i1 // T.int64(8)])
T.writes(compute[v_i0, v_i1])
compute[v_i0, v_i1] = T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight1[v_i0, v_i1 // T.int64(8)], T.Cast("uint32", v_i1 % T.int64(8) * T.int64(4))), T.uint32(15)))
for i0, i1 in T.grid(T.int64(2048), T.int64(512)):
with T.block("dequantize"):
v_i0, v_i1 = T.axis.remap("SS", [i0, i1])
T.reads(compute[v_i0, v_i1], gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale1[v_i0, v_i1 // T.int64(32)])
T.writes(dequantize[v_i0, v_i1])
dequantize[v_i0, v_i1] = (compute[v_i0, v_i1] - T.float16(7.0)) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale1[v_i0, v_i1 // T.int64(32)]
@T.prim_func(private=True)
def dequantize4(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight1: T.Buffer((T.int64(512), T.int64(256)), "uint32"), gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale1: T.Buffer((T.int64(512), T.int64(64)), "float16"), dequantize: T.Buffer((T.int64(512), T.int64(2048)), "float16")):
T.func_attr({"target": T.target({"keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)})
# with T.block("root"):
compute = T.alloc_buffer((T.int64(512), T.int64(2048)), "float16")
for i0, i1 in T.grid(T.int64(512), T.int64(2048)):
with T.block("compute"):
v_i0, v_i1 = T.axis.remap("SS", [i0, i1])
T.reads(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight1[v_i0, v_i1 // T.int64(8)])
T.writes(compute[v_i0, v_i1])
compute[v_i0, v_i1] = T.Cast("float16", T.bitwise_and(T.shift_right(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight1[v_i0, v_i1 // T.int64(8)], T.Cast("uint32", v_i1 % T.int64(8) * T.int64(4))), T.uint32(15)))
for i0, i1 in T.grid(T.int64(512), T.int64(2048)):
with T.block("dequantize"):
v_i0, v_i1 = T.axis.remap("SS", [i0, i1])
T.reads(compute[v_i0, v_i1], gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale1[v_i0, v_i1 // T.int64(32)])
T.writes(dequantize[v_i0, v_i1])
dequantize[v_i0, v_i1] = (compute[v_i0, v_i1] - T.float16(7.0)) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale1[v_i0, v_i1 // T.int64(32)]
@T.prim_func
def fused_rope(var_qkv: T.handle, var_position_map: T.handle, var_q: T.handle, var_k: T.handle, var_v: T.handle, apply_rope: T.int32):
T.func_attr({"op_pattern": 8, "target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)})
seq_len = T.int32()
qkv = T.match_buffer(var_qkv, (seq_len, 24, 64), "float16")
position_map = T.match_buffer(var_position_map, (seq_len,), "int32", offset_factor=1)
q = T.match_buffer(var_q, (seq_len, 8, 64), "float16")
k = T.match_buffer(var_k, (seq_len, 8, 64), "float16")
v = T.match_buffer(var_v, (seq_len, 8, 64), "float16")
# with T.block("root"):
for iters_0, iters_1, iters_2 in T.grid(seq_len, 24, 64):
with T.block("llama_fused_rope"):
s, h, d = T.axis.remap("SSS", [iters_0, iters_1, iters_2])
T.reads(position_map[s], qkv[s, h, d - 8:d - 8 + 17])
T.writes(q[s, h, d], k[s, h - 8, d], v[s, h - 16, d])
if h < 8:
freq = T.float32()
q[s, h, d] = T.if_then_else(apply_rope > 0 and d < 16, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", qkv[s, h, d]) + T.sin(freq) * T.Cast("float32", T.if_then_else(d < 8, qkv[s, h, d + 8] * T.float16(-1.0), qkv[s, h, d - 8]))), where={freq: T.Cast("float32", position_map[s]) / T.pow(T.float32(10000.0), T.Cast("float32", d * 2 % 16) / T.float32(16.0))}), qkv[s, h, d])
else:
if h < 16:
freq = T.float32()
k[s, h - 8, d] = T.if_then_else(apply_rope > 0 and d < 16, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", qkv[s, h, d]) + T.sin(freq) * T.Cast("float32", T.if_then_else(d < 8, qkv[s, h, d + 8] * T.float16(-1.0), qkv[s, h, d - 8]))), where={freq: T.Cast("float32", position_map[s]) / T.pow(T.float32(10000.0), T.Cast("float32", d * 2 % 16) / T.float32(16.0))}), qkv[s, h, d])
else:
v[s, h - 16, d] = qkv[s, h, d]
@T.prim_func
def gather_probs(var_src: T.handle, var_indices: T.handle, var_dst: T.handle):
T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)})
m, n = T.int32(is_size_var=True), T.int32(is_size_var=True)
src = T.match_buffer(var_src, (m, n))
batch_size = T.int32(is_size_var=True)
indices = T.match_buffer(var_indices, (batch_size,), "int32")
dst = T.match_buffer(var_dst, (batch_size, n))
# with T.block("root"):
for b, j in T.grid(batch_size, n):
with T.block("gather_2d"):
vb, vj = T.axis.remap("SS", [b, j])
T.reads(src[indices[vb], vj], indices[vb])
T.writes(dst[vb, vj])
dst[vb, vj] = src[indices[vb], vj]
@T.prim_func(private=True)
def index(var_layer_norm12: T.handle, index: T.Buffer((T.int64(1), T.int64(1), T.int64(512)), "float16")):
T.func_attr({"target": T.target({"keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)})
seq_len = T.int64()
layer_norm12 = T.match_buffer(var_layer_norm12, (T.int64(1), seq_len, T.int64(512)), "float16")
# with T.block("root"):
for i, _, k in T.grid(T.int64(1), T.int64(1), T.int64(512)):
with T.block("index"):
v_i, v__, v_k = T.axis.remap("SSS", [i, _, k])
T.reads(layer_norm12[v_i, seq_len - T.int64(1), v_k])
T.writes(index[v_i, v__, v_k])
index[v_i, v__, v_k] = layer_norm12[v_i, seq_len - T.int64(1), v_k]
@T.prim_func
def merge_state_inplace(v: T.handle, s: T.handle, v_other: T.handle, s_other: T.handle):
T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1})
N, H, D = T.int32(is_size_var=True), T.int32(is_size_var=True), T.int32(is_size_var=True)
V = T.match_buffer(v, (N, H, D), "float16")
S = T.match_buffer(s, (N, H))
V_other = T.match_buffer(v_other, (N, H, D), "float16")
S_other = T.match_buffer(s_other, (N, H))
# with T.block("root"):
for bx in T.thread_binding(N, thread="blockIdx.x"):
for by in T.thread_binding(1, thread="blockIdx.y"):
for ty in T.thread_binding(8, thread="threadIdx.y"):
for tx in T.thread_binding(16, thread="threadIdx.x"):
with T.block("merge"):
T.reads(S[bx, ty + by * 8], S_other[bx, ty + by * 8], V[bx, ty + by * 8, tx * 4:tx * 4 + 4], V_other[bx, ty + by * 8, tx * 4:tx * 4 + 4])
T.writes(V[bx, ty + by * 8, tx * 4:tx * 4 + 4], S[bx, ty + by * 8])
s_val = T.alloc_buffer((1,), scope="local")
s_other_val = T.alloc_buffer((1,), scope="local")
s_max = T.alloc_buffer((1,), scope="local")
scale = T.alloc_buffer((1,), scope="local")
other_scale = T.alloc_buffer((1,), scope="local")
v_vec = T.alloc_buffer((4,), "float16", scope="local")
v_other_vec = T.alloc_buffer((4,), "float16", scope="local")
s_val[0] = S[bx, ty + by * 8]
s_other_val[0] = S_other[bx, ty + by * 8]
s_max[0] = T.max(s_val[0], s_other_val[0])
s_val[0] = T.exp2(s_val[0] - s_max[0])
s_other_val[0] = T.exp2(s_other_val[0] - s_max[0])
scale[0] = s_val[0] / (s_val[0] + s_other_val[0])
other_scale[0] = s_other_val[0] / (s_val[0] + s_other_val[0])
for vec in T.vectorized(4):
v_vec[vec] = V[bx, ty + by * 8, tx * 4 + vec]
for vec in T.vectorized(4):
v_other_vec[vec] = V_other[bx, ty + by * 8, tx * 4 + vec]
for vec in range(4):
v_vec[vec] = T.Cast("float16", T.Cast("float32", v_vec[vec]) * scale[0] + T.Cast("float32", v_other_vec[vec]) * other_scale[0])
for vec in T.vectorized(4):
V[bx, ty + by * 8, tx * 4 + vec] = v_vec[vec]
S[bx, ty + by * 8] = T.log2(s_val[0] + s_other_val[0]) + s_max[0]
@T.prim_func
def scatter_probs(var_src: T.handle, var_indices: T.handle, var_dst: T.handle):
T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)})
batch_size, n = T.int32(is_size_var=True), T.int32(is_size_var=True)
src = T.match_buffer(var_src, (batch_size, n))
indices = T.match_buffer(var_indices, (batch_size,), "int32")
m = T.int32(is_size_var=True)
dst = T.match_buffer(var_dst, (m, n))
# with T.block("root"):
for b, j in T.grid(batch_size, n):
with T.block("scatter_2d"):
vb, vj = T.axis.remap("SS", [b, j])
T.reads(src[vb, vj], indices[vb])
T.writes(dst[indices[vb], vj])
dst[indices[vb], vj] = src[vb, vj]
@T.prim_func
def softmax_with_chunked_sum(var_A: T.handle, var_temperature: T.handle, var_chunked_sum: T.handle, var_chunked_max: T.handle, var_softmax: T.handle):
T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1, "tir.noalias": T.bool(True)})
batch_size, vocab_size = T.int64(is_size_var=True), T.int64(is_size_var=True)
A = T.match_buffer(var_A, (batch_size, vocab_size))
temperature = T.match_buffer(var_temperature, (batch_size,))
num_chunks = T.int64(is_size_var=True)
chunked_sum = T.match_buffer(var_chunked_sum, (batch_size, num_chunks))
chunked_max = T.match_buffer(var_chunked_max, (batch_size, num_chunks))
softmax = T.match_buffer(var_softmax, (batch_size, vocab_size))
# with T.block("root"):
temp_max_shared = T.alloc_buffer((batch_size,), scope="shared")
temp_sum_shared = T.alloc_buffer((batch_size,), scope="shared")
for l0_l1_fused in T.thread_binding(batch_size * num_chunks, thread="blockIdx.x"):
for ax0_1 in T.thread_binding(T.int64(32), thread="threadIdx.x"):
for ax0_0 in T.serial((num_chunks + T.int64(31)) // T.int64(32), annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}):
with T.block("max"):
v0 = T.axis.spatial(batch_size, l0_l1_fused % (num_chunks * batch_size) // num_chunks)
v1 = T.axis.reduce(num_chunks, ax0_0 * T.int64(32) + ax0_1)
T.where(ax0_0 * T.int64(32) + ax0_1 < num_chunks)
T.reads(chunked_max[v0, v1])
T.writes(temp_max_shared[v0])
with T.init():
temp_max_shared[v0] = T.float32(-340282346638528859811704183484516925440.0)
temp_max_shared[v0] = T.max(temp_max_shared[v0], chunked_max[v0, v1])
for ax0_1 in T.thread_binding(T.int64(32), thread="threadIdx.x"):
for ax0_0 in T.serial((num_chunks + T.int64(31)) // T.int64(32), annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}):
with T.block("sum_exp"):
v0 = T.axis.spatial(batch_size, l0_l1_fused % (num_chunks * batch_size) // num_chunks)
v1 = T.axis.reduce(num_chunks, ax0_0 * T.int64(32) + ax0_1)
T.where(ax0_0 * T.int64(32) + ax0_1 < num_chunks)
T.reads(temperature[v0], chunked_sum[v0, v1], chunked_max[v0, v1], temp_max_shared[v0])
T.writes(temp_sum_shared[v0])
with T.init():
temp_sum_shared[v0] = T.float32(0.0)
temp_sum_shared[v0] = temp_sum_shared[v0] + T.Select(temperature[v0] > T.float32(1.0000000000000001e-05), T.exp(chunked_sum[v0, v1] + chunked_max[v0, v1] - temp_max_shared[v0]), T.Cast("float32", chunked_max[v0, v1] == temp_max_shared[v0]) * chunked_sum[v0, v1])
for l2_0 in T.serial(T.int64(16), annotations={"pragma_auto_unroll_max_step": 64, "pragma_unroll_explicit": 1}):
for l2_1 in T.thread_binding(T.int64(8), thread="threadIdx.y"):
for l2_2 in T.thread_binding(T.int64(32), thread="threadIdx.x"):
with T.block("log_pad"):
v0 = T.axis.spatial(batch_size, l0_l1_fused % (num_chunks * batch_size) // num_chunks)
v1 = T.axis.spatial(num_chunks, l0_l1_fused % num_chunks)
v2 = T.axis.spatial(T.int64(4096), l2_0 * T.int64(256) + l2_1 * T.int64(32) + l2_2)
T.reads(temperature[v0], A[v0, v1 * T.int64(4096) + v2], temp_sum_shared[v0], temp_max_shared[v0])
T.writes(softmax[v0, v1 * T.int64(4096) + v2])
if v1 * T.int64(4096) + v2 < vocab_size:
softmax[v0, v1 * T.int64(4096) + v2] = T.if_then_else(temperature[v0] > T.float32(1.0000000000000001e-05), T.exp(A[v0, v1 * T.int64(4096) + v2] / temperature[v0] - (T.log(temp_sum_shared[v0]) + temp_max_shared[v0])), T.Cast("float32", A[v0, v1 * T.int64(4096) + v2] == temp_max_shared[v0]) / temp_sum_shared[v0])
@T.prim_func
def tir_kv_cache_debug_get_kv(var_pages: T.handle, var_position_map: T.handle, var_k_data: T.handle, var_v_data: T.handle, layer_id: T.int64):
T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)})
num_pages, page_size = T.int64(), T.int64(is_size_var=True)
pages = T.match_buffer(var_pages, (num_pages, 2, 8, page_size, 64), "float16", offset_factor=1)
seqlen = T.int64(is_size_var=True)
position_map = T.match_buffer(var_position_map, (seqlen,), "int32", offset_factor=1)
k_data = T.match_buffer(var_k_data, (6, seqlen, 8, 64), "float16")
v_data = T.match_buffer(var_v_data, (6, seqlen, 8, 64), "float16")
# with T.block("root"):
for p, h, d in T.grid(seqlen, 8, 64):
with T.block("copy0"):
vp, vh, vd = T.axis.remap("SSS", [p, h, d])
T.reads(position_map[vp], pages[T.Cast("int64", position_map[vp]) // page_size, 0:2, vh, T.Cast("int64", position_map[vp]) % page_size, vd])
T.writes(k_data[layer_id, vp, vh, vd], v_data[layer_id, vp, vh, vd])
position: T.int32 = position_map[vp]
k_data[layer_id, vp, vh, vd] = pages[T.Cast("int64", position) // page_size, 0, vh, T.Cast("int64", position) % page_size, vd]
v_data[layer_id, vp, vh, vd] = pages[T.Cast("int64", position) // page_size, 1, vh, T.Cast("int64", position) % page_size, vd]
@T.prim_func
def tir_kv_cache_transpose_append(var_pages: T.handle, var_k_data: T.handle, var_v_data: T.handle, var_position_map: T.handle):
T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.noalias": T.bool(True)})
num_pages = T.int64()
pages = T.match_buffer(var_pages, (num_pages, 2, 8, 16, 64), "float16", offset_factor=1)
ntoken = T.int64(is_size_var=True)
k_data = T.match_buffer(var_k_data, (ntoken, 8, 64), "float16")
v_data = T.match_buffer(var_v_data, (ntoken, 8, 64), "float16")
position_map = T.match_buffer(var_position_map, (ntoken,), "int32", offset_factor=1)
# with T.block("root"):
for global_pos, h, f in T.grid(ntoken, 8, 64):
if position_map[global_pos] != -1:
with T.block("k_transpose_append"):
vgpos, vh, vf = T.axis.remap("SSS", [global_pos, h, f])
T.reads(position_map[vgpos], k_data[vgpos, vh, vf])
T.writes(pages[position_map[vgpos] // 16, 0, vh, position_map[vgpos] % 16, vf])
position: T.int32 = position_map[vgpos]
pages[position // 16, 0, vh, position % 16, vf] = k_data[vgpos, vh, vf]
with T.block("v_transpose_append"):
vgpos, vh, vf = T.axis.remap("SSS", [global_pos, h, f])
T.reads(position_map[vgpos], v_data[vgpos, vh, vf])
T.writes(pages[position_map[vgpos] // 16, 1, vh, position_map[vgpos] % 16, vf])
position: T.int32 = position_map[vgpos]
pages[position // 16, 1, vh, position % 16, vf] = v_data[vgpos, vh, vf]
@T.prim_func
def tree_attn_paged_kv(_0: T.int32, var_q: T.handle, var_q_indptr: T.handle, var_pages: T.handle, var_page_indptr: T.handle, var_page_values: T.handle, var_length_info: T.handle, var_k_rope_pos_offset: T.handle, var_q_rope_position: T.handle, var_output: T.handle, var_lse: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32, tree_order_indptr_handle: T.handle, tree_order_handle: T.handle):
T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1})
total_len = T.int32(is_size_var=True)
q = T.match_buffer(var_q, (total_len, 8, 64), "float16")
batch_size = T.int32(is_size_var=True)
q_indptr = T.match_buffer(var_q_indptr, (batch_size + 1,), "int32", offset_factor=1)
max_num_pages = T.int32(is_size_var=True)
pages = T.match_buffer(var_pages, (max_num_pages, 2, 8, 32, 64), "float16")
page_indptr = T.match_buffer(var_page_indptr, (batch_size + 1,), "int32", offset_factor=1)
nnz_pages = T.int32(is_size_var=True)
page_values = T.match_buffer(var_page_values, (nnz_pages,), "int32", offset_factor=1)
length_info = T.match_buffer(var_length_info, (batch_size,), "int32", offset_factor=1)
k_rope_pos_offset = T.match_buffer(var_k_rope_pos_offset, (batch_size,), "int32", offset_factor=1)
q_rope_position = T.match_buffer(var_q_rope_position, (total_len,), "int32", offset_factor=1)
output = T.match_buffer(var_output, (total_len, 8, 64), "float16")
lse = T.match_buffer(var_lse, (total_len, 8))
tree_order_indptr = T.match_buffer(tree_order_indptr_handle, (batch_size + 1,), "int32", offset_factor=1)
total_tree_order_len = T.int32(is_size_var=True)
tree_order = T.match_buffer(tree_order_handle, (total_tree_order_len, 2), "int32", offset_factor=1)
# with T.block("root"):
assert rotary_mode == 0, "Inline rotary mode is not supported in tree attention."
for lbx in T.thread_binding(16, thread="blockIdx.x"):
for lby in T.thread_binding(8, thread="blockIdx.y"):
for lty in T.thread_binding(4, thread="threadIdx.y"):
for ltx in T.thread_binding(32, thread="threadIdx.x"):
with T.block("attn"):
bx, by, ty, tx = T.axis.remap("SSSS", [lbx, lby, lty, ltx])
T.reads()
T.writes()
tile_id = T.alloc_buffer((1,), "int32", scope="local")
batch_idx = T.alloc_buffer((1,), "int32", scope="local")
batch_tiles = T.alloc_buffer((1,), "int32", scope="local")
batch_rows = T.alloc_buffer((1,), "int32", scope="local")
iterator = T.alloc_buffer((1,), "int32", scope="local")
kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local")
Q_smem = T.alloc_buffer((32, 64), "float16", scope="shared")
K_smem = T.alloc_buffer((32, 64), "float16", scope="shared")
V_smem = T.alloc_buffer((32, 64), "float16", scope="shared")
S_smem = T.alloc_buffer((32, 32), scope="shared")
S_local = T.alloc_buffer((32, 32), scope="local")
O_local = T.alloc_buffer((32, 64), scope="local")
m_smem = T.alloc_buffer((32,), scope="shared")
m_prev_smem = T.alloc_buffer((32,), scope="shared")
d_smem = T.alloc_buffer((32,), scope="shared")
m_new = T.alloc_buffer((1,), scope="local")
m_prev = T.alloc_buffer((1,), scope="local")
d_new = T.alloc_buffer((1,), scope="local")
tile_id[0] = bx
batch_idx[0] = 0
batch_rows[0] = q_indptr[1] - q_indptr[0]
batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32
while T.tvm_thread_invariant(batch_idx[0] < batch_size):
while tile_id[0] >= batch_tiles[0] and batch_idx[0] < batch_size:
tile_id[0] = tile_id[0] - batch_tiles[0]
batch_idx[0] = batch_idx[0] + 1
if batch_idx[0] < batch_size:
b_idx: T.int32 = batch_idx[0]
batch_rows[0] = q_indptr[b_idx + 1] - q_indptr[b_idx]
batch_tiles[0] = (batch_rows[0] + 32 - 1) // 32
if T.tvm_thread_invariant(batch_idx[0] < batch_size):
b_idx: T.int32 = batch_idx[0]
LH_start: T.int32 = tile_id[0] * 32
q_indptr_val: T.int32 = q_indptr[b_idx]
cur_page_indptr_begin: T.int32 = page_indptr[b_idx]
cur_page_indptr_end: T.int32 = page_indptr[b_idx + 1]
kv_chunk_len[0] = T.if_then_else(cur_page_indptr_begin != cur_page_indptr_end, (cur_page_indptr_end - cur_page_indptr_begin - 1) * 32 + length_info[b_idx], 0)
T.tvm_storage_sync("shared")
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
if row < 32:
m_smem[row] = T.float32(-50000.0)
d_smem[row] = T.float32(1.0)
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for li_1, lj_1 in T.grid(4, 4):
with T.block("O_init"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1)
j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1)
T.reads()
T.writes(O_local[i, j])
O_local[i, j] = T.float32(0.0)
T.tvm_storage_sync("shared")
for li_lj_fused_0 in range(4):
for li_lj_fused_1 in T.thread_binding(4, thread="threadIdx.y"):
for li_lj_fused_2 in T.thread_binding(32, thread="threadIdx.x"):
for li_lj_fused_3 in T.vectorized(4):
with T.block("Q_load"):
i = T.axis.spatial(32, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) // 64)
j = T.axis.spatial(64, (li_lj_fused_0 * 512 + li_lj_fused_1 * 128 + li_lj_fused_2 * 4 + li_lj_fused_3) % 64)
T.reads()
T.writes()
cur_L: T.int32 = q_indptr_val + (LH_start + i)
cur_H_qo: T.int32 = by
if cur_L < q_indptr[b_idx + 1]:
freq = T.float32()
Q_smem[i, j] = T.if_then_else(rotary_mode == 1, T.Let(T.Cast("float16", T.cos(freq) * T.Cast("float32", q[cur_L, cur_H_qo, j]) + T.sin(freq) * T.Cast("float32", T.if_then_else(j < 32, q[cur_L, cur_H_qo, j + 32] * T.float16(-1.0), q[cur_L, cur_H_qo, j - 32]))), where={freq: T.Cast("float32", q_rope_position[cur_L]) * rope_scale / T.pow(rope_theta, T.Cast("float32", j * 2 % 64) / T.float32(64.0))}), q[cur_L, cur_H_qo, j])
else:
Q_smem[i, j] = T.float16(0.0)
T.tvm_storage_sync("shared")
for iterator_1 in range((kv_chunk_len[0] + 31) // 32):
L_kv_start: T.int32 = iterator_1 * 32
for lz_ly_fused_0 in range(4):
for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"):
for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"):
for lz_ly_fused_3 in T.vectorized(4):
with T.block("K_load"):
i = T.axis.spatial(32, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64)
j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64)
T.reads()
T.writes()
cur_L: T.int32 = L_kv_start + i
if cur_L < kv_chunk_len[0]:
seq_offset: T.int32 = cur_L
page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 32]
page_offset: T.int32 = seq_offset % 32
K_smem[i, j] = pages[page_no, 0, by, page_offset, j]
else:
K_smem[i, j] = T.float16(0.0)
T.tvm_storage_sync("shared")
for lz_ly_fused_0 in range(4):
for lz_ly_fused_1 in T.thread_binding(4, thread="threadIdx.y"):
for lz_ly_fused_2 in T.thread_binding(32, thread="threadIdx.x"):
for lz_ly_fused_3 in T.vectorized(4):
with T.block("V_load"):
i = T.axis.spatial(32, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) // 64)
j = T.axis.spatial(64, (lz_ly_fused_0 * 512 + lz_ly_fused_1 * 128 + lz_ly_fused_2 * 4 + lz_ly_fused_3) % 64)
T.reads()
T.writes()
cur_L: T.int32 = L_kv_start + i
if cur_L < kv_chunk_len[0]:
seq_offset: T.int32 = cur_L
page_no: T.int32 = page_values[cur_page_indptr_begin + seq_offset // 32]
page_offset: T.int32 = seq_offset % 32
V_smem[i, j] = pages[page_no, 1, by, page_offset, j]
else:
V_smem[i, j] = T.float16(0.0)
T.tvm_storage_sync("shared")
with T.block(""):
T.reads(Q_smem[0:32, 0:64], K_smem[0:32, 0:64])
T.writes(S_local[0:32, 0:32])
for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"):
for li_1_init, lj_1_init in T.grid(2, 4):
with T.block("S_gemm_init"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 8 * 2 + li_1_init)
j = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 8 * 4 + lj_1_init)
T.reads()
T.writes(S_local[i, j])
S_local[i, j] = T.float32(0.0)
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for lk_0, li_1, lj_1, lk_1 in T.grid(8, 2, 4, 8):
with T.block("S_gemm_update"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1)
j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1)
k = T.axis.reduce(64, lk_0 * 8 + lk_1)
T.reads(S_local[i, j], Q_smem[i, k], K_smem[j, k])
T.writes(S_local[i, j])
S_local[i, j] = S_local[i, j] + T.Cast("float32", Q_smem[i, k]) * T.Cast("float32", K_smem[j, k]) * attn_score_scaling_factor * T.float32(0.18033688011112042)
T.tvm_storage_sync("shared")
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for li_1, lj_1 in T.grid(2, 4):
with T.block("S_store"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 8 * 2 + li_1)
j = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 8 * 4 + lj_1)
T.reads(S_local[i, j])
T.writes(S_smem[i, j])
S_smem[i, j] = S_local[i, j]
T.tvm_storage_sync("shared")
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
if row < 32:
with T.block("update1"):
T.reads(m_smem[row], kv_chunk_len[0], tree_order_indptr[b_idx:b_idx + 2], tree_order[T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]):T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], m_new[i], S_smem[row, 0:32], d_smem[row], m_prev[i])
T.writes(m_prev[i], m_new[i], d_new[i])
m_prev[i] = m_smem[row]
m_new[i] = m_smem[row]
row_: T.int32 = LH_start + row
for j in range(32):
if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) or tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 0] and tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 1]):
m_new[i] = T.max(m_new[i], S_smem[row, j])
d_new[i] = d_smem[row] * T.exp2(m_prev[i] - m_new[i])
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
with T.block("update"):
T.reads(kv_chunk_len[0], tree_order_indptr[b_idx:b_idx + 2], tree_order[T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]):T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0]) + (T.max(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] + 31 - kv_chunk_len[0]) + 1 - T.min(LH_start + row + tree_order_indptr[b_idx + 1] + q_indptr[b_idx] - q_indptr[b_idx + 1], L_kv_start + tree_order_indptr[b_idx + 1] - kv_chunk_len[0])), 0:2], q_indptr[b_idx:b_idx + 2], S_smem[row, 0:32], m_new[i])
T.writes(S_smem[row, 0:32])
for j in range(32):
if row < 32:
row_: T.int32 = LH_start + row
if L_kv_start + j < kv_chunk_len[0] and (L_kv_start + j < kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) or tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] >= tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 0] and tree_order[tree_order_indptr[b_idx] + (row_ + (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]) - (q_indptr[b_idx + 1] - q_indptr[b_idx])), 0] < tree_order[tree_order_indptr[b_idx] + (L_kv_start + j - (kv_chunk_len[0] - (tree_order_indptr[b_idx + 1] - tree_order_indptr[b_idx]))), 1]):
S_smem[row, j] = T.exp2(S_smem[row, j] - m_new[i])
else:
S_smem[row, j] = T.exp2(T.float32(-50000.0) - m_new[i])
for i in range(1):
row: T.int32 = i * 32 * 4 + ty * 32 + tx
if row < 32:
with T.block("update"):
T.reads(d_new[i], S_smem[row, 0:32], m_new[i], m_prev[i])
T.writes(d_new[i], m_smem[row], d_smem[row], m_prev_smem[row])
for j in range(32):
d_new[i] = d_new[i] + S_smem[row, j]
m_smem[row] = m_new[i]
d_smem[row] = d_new[i]
m_prev_smem[row] = m_prev[i]
T.tvm_storage_sync("shared")
with T.block(""):
T.reads(m_prev_smem[0:32], m_smem[0:32], S_smem[0:32, 0:32], V_smem[0:32, 0:64])
T.writes(O_local[0:32, 0:64])
for li_0_lj_0_fused_0_init in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1_init in T.thread_binding(32, thread="threadIdx.x"):
for li_1_init, lj_1_init in T.grid(4, 4):
with T.block("O_gemm_init"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) // 16 * 4 + li_1_init)
j = T.axis.spatial(64, (li_0_lj_0_fused_0_init * 32 + li_0_lj_0_fused_1_init) % 16 * 4 + lj_1_init)
T.reads()
T.writes(O_local[i, j])
O_local[i, j] = O_local[i, j] * T.exp2(m_prev_smem[i] - m_smem[i])
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for lk_0, lk_1, li_1, lj_1 in T.grid(4, 8, 4, 4):
with T.block("O_gemm_update"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1)
j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1)
k = T.axis.reduce(32, lk_0 * 8 + lk_1)
T.reads(O_local[i, j], m_prev_smem[i], m_smem[i], S_smem[i, k], V_smem[k, j])
T.writes(O_local[i, j])
O_local[i, j] = O_local[i, j] + S_smem[i, k] * T.Cast("float32", V_smem[k, j])
for li_0_lj_0_fused_0 in T.thread_binding(4, thread="threadIdx.y"):
for li_0_lj_0_fused_1 in T.thread_binding(32, thread="threadIdx.x"):
for li_1, lj_1 in T.grid(4, 4):
with T.block("O_store"):
i = T.axis.spatial(32, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) // 16 * 4 + li_1)
j = T.axis.spatial(64, (li_0_lj_0_fused_0 * 32 + li_0_lj_0_fused_1) % 16 * 4 + lj_1)
T.reads(q_indptr[b_idx:b_idx + 2], O_local[i, j], d_smem[i])
T.writes(output[q_indptr[b_idx] + (LH_start + i), by, j])
cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i)
cur_H_qo: T.int32 = by
if cur_L < q_indptr[b_idx + 1]:
output[cur_L, cur_H_qo, j] = T.Cast("float16", O_local[i, j] / d_smem[i])
for li_0 in range(1):
for li_1 in T.thread_binding(4, thread="threadIdx.y"):
for li_2 in T.thread_binding(32, thread="threadIdx.x"):
with T.block("lse_store"):
i = T.axis.spatial(32, li_0 * 128 + li_1 * 32 + li_2)
T.where((li_0 * 4 + li_1) * 32 + li_2 < 32)
T.reads(q_indptr[b_idx:b_idx + 2], m_smem[i], d_smem[i])
T.writes(lse[q_indptr[b_idx] + (LH_start + i), by])
cur_L: T.int32 = q_indptr[b_idx] + (LH_start + i)
cur_H_qo: T.int32 = by
if cur_L < q_indptr[b_idx + 1]:
lse[cur_L, cur_H_qo] = m_smem[i] + T.log2(d_smem[i])
tile_id[0] = tile_id[0] + 16
@R.function
def alloc_embedding_tensor() -> R.Tensor((2048, 512), dtype="float16"):
R.func_attr({"relax.memory_plan_dynamic_func_output": True})
gv: R.Tensor((2048, 512), dtype="float16") = R.builtin.alloc_tensor(R.shape([2048, 512]), R.dtype("float16"), R.prim_value(0), R.str("global"))
return gv
@R.function
def batch_decode(input_embeds: R.Tensor(("batch_size", 1, 512), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"))) -> R.Tuple(R.Tensor(("batch_size", 1, "vocab_size"), dtype="float32"), R.Object):
batch_size = T.int64()
vocab_size = T.int64()
R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.memory_plan_dynamic_func_output": True, "relax.rewrite_cuda_graph.capture_symbolic_vars": ["batch_size"], "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}})
cls = Module
with R.dataflow():
gpt_neox_embed_in_q_weight4: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[0]
gpt_neox_embed_in_q_scale4: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[1]
gpt_neox_layers_0_input_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[2]
gpt_neox_layers_0_input_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[3]
gpt_neox_layers_0_post_attention_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[4]
gpt_neox_layers_0_post_attention_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[5]
gpt_neox_layers_0_attention_query_key_value_q_weight4: R.Tensor((1536, 64), dtype="uint32") = packed_params[6]
gpt_neox_layers_0_attention_query_key_value_q_scale4: R.Tensor((1536, 16), dtype="float16") = packed_params[7]
gpt_neox_layers_0_attention_query_key_value_bias4: R.Tensor((1536,), dtype="float16") = packed_params[8]
gpt_neox_layers_0_attention_dense_q_weight4: R.Tensor((512, 64), dtype="uint32") = packed_params[9]
gpt_neox_layers_0_attention_dense_q_scale4: R.Tensor((512, 16), dtype="float16") = packed_params[10]
gpt_neox_layers_0_attention_dense_bias4: R.Tensor((512,), dtype="float16") = packed_params[11]
gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4: R.Tensor((2048, 64), dtype="uint32") = packed_params[12]
gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4: R.Tensor((2048, 16), dtype="float16") = packed_params[13]
gpt_neox_layers_0_mlp_dense_h_to_4h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[14]
gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4: R.Tensor((512, 256), dtype="uint32") = packed_params[15]
gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4: R.Tensor((512, 64), dtype="float16") = packed_params[16]
gpt_neox_layers_0_mlp_dense_4h_to_h_bias4: R.Tensor((512,), dtype="float32") = packed_params[17]
gpt_neox_layers_1_input_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[18]
gpt_neox_layers_1_input_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[19]
gpt_neox_layers_1_post_attention_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[20]
gpt_neox_layers_1_post_attention_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[21]
gpt_neox_layers_1_attention_query_key_value_q_weight4: R.Tensor((1536, 64), dtype="uint32") = packed_params[22]
gpt_neox_layers_1_attention_query_key_value_q_scale4: R.Tensor((1536, 16), dtype="float16") = packed_params[23]
gpt_neox_layers_1_attention_query_key_value_bias4: R.Tensor((1536,), dtype="float16") = packed_params[24]
gpt_neox_layers_1_attention_dense_q_weight4: R.Tensor((512, 64), dtype="uint32") = packed_params[25]
gpt_neox_layers_1_attention_dense_q_scale4: R.Tensor((512, 16), dtype="float16") = packed_params[26]
gpt_neox_layers_1_attention_dense_bias4: R.Tensor((512,), dtype="float16") = packed_params[27]
gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight4: R.Tensor((2048, 64), dtype="uint32") = packed_params[28]
gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale4: R.Tensor((2048, 16), dtype="float16") = packed_params[29]
gpt_neox_layers_1_mlp_dense_h_to_4h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[30]
gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight4: R.Tensor((512, 256), dtype="uint32") = packed_params[31]
gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale4: R.Tensor((512, 64), dtype="float16") = packed_params[32]
gpt_neox_layers_1_mlp_dense_4h_to_h_bias4: R.Tensor((512,), dtype="float32") = packed_params[33]
gpt_neox_layers_2_input_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[34]
gpt_neox_layers_2_input_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[35]
gpt_neox_layers_2_post_attention_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[36]
gpt_neox_layers_2_post_attention_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[37]
gpt_neox_layers_2_attention_query_key_value_q_weight4: R.Tensor((1536, 64), dtype="uint32") = packed_params[38]
gpt_neox_layers_2_attention_query_key_value_q_scale4: R.Tensor((1536, 16), dtype="float16") = packed_params[39]
gpt_neox_layers_2_attention_query_key_value_bias4: R.Tensor((1536,), dtype="float16") = packed_params[40]
gpt_neox_layers_2_attention_dense_q_weight4: R.Tensor((512, 64), dtype="uint32") = packed_params[41]
gpt_neox_layers_2_attention_dense_q_scale4: R.Tensor((512, 16), dtype="float16") = packed_params[42]
gpt_neox_layers_2_attention_dense_bias4: R.Tensor((512,), dtype="float16") = packed_params[43]
gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight4: R.Tensor((2048, 64), dtype="uint32") = packed_params[44]
gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale4: R.Tensor((2048, 16), dtype="float16") = packed_params[45]
gpt_neox_layers_2_mlp_dense_h_to_4h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[46]
gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight4: R.Tensor((512, 256), dtype="uint32") = packed_params[47]
gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale4: R.Tensor((512, 64), dtype="float16") = packed_params[48]
gpt_neox_layers_2_mlp_dense_4h_to_h_bias4: R.Tensor((512,), dtype="float32") = packed_params[49]
gpt_neox_layers_3_input_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[50]
gpt_neox_layers_3_input_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[51]
gpt_neox_layers_3_post_attention_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[52]
gpt_neox_layers_3_post_attention_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[53]
gpt_neox_layers_3_attention_query_key_value_q_weight4: R.Tensor((1536, 64), dtype="uint32") = packed_params[54]
gpt_neox_layers_3_attention_query_key_value_q_scale4: R.Tensor((1536, 16), dtype="float16") = packed_params[55]
gpt_neox_layers_3_attention_query_key_value_bias4: R.Tensor((1536,), dtype="float16") = packed_params[56]
gpt_neox_layers_3_attention_dense_q_weight4: R.Tensor((512, 64), dtype="uint32") = packed_params[57]
gpt_neox_layers_3_attention_dense_q_scale4: R.Tensor((512, 16), dtype="float16") = packed_params[58]
gpt_neox_layers_3_attention_dense_bias4: R.Tensor((512,), dtype="float16") = packed_params[59]
gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight4: R.Tensor((2048, 64), dtype="uint32") = packed_params[60]
gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale4: R.Tensor((2048, 16), dtype="float16") = packed_params[61]
gpt_neox_layers_3_mlp_dense_h_to_4h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[62]
gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight4: R.Tensor((512, 256), dtype="uint32") = packed_params[63]
gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale4: R.Tensor((512, 64), dtype="float16") = packed_params[64]
gpt_neox_layers_3_mlp_dense_4h_to_h_bias4: R.Tensor((512,), dtype="float32") = packed_params[65]
gpt_neox_layers_4_input_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[66]
gpt_neox_layers_4_input_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[67]
gpt_neox_layers_4_post_attention_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[68]
gpt_neox_layers_4_post_attention_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[69]
gpt_neox_layers_4_attention_query_key_value_q_weight4: R.Tensor((1536, 64), dtype="uint32") = packed_params[70]
gpt_neox_layers_4_attention_query_key_value_q_scale4: R.Tensor((1536, 16), dtype="float16") = packed_params[71]
gpt_neox_layers_4_attention_query_key_value_bias4: R.Tensor((1536,), dtype="float16") = packed_params[72]
gpt_neox_layers_4_attention_dense_q_weight4: R.Tensor((512, 64), dtype="uint32") = packed_params[73]
gpt_neox_layers_4_attention_dense_q_scale4: R.Tensor((512, 16), dtype="float16") = packed_params[74]
gpt_neox_layers_4_attention_dense_bias4: R.Tensor((512,), dtype="float16") = packed_params[75]
gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight4: R.Tensor((2048, 64), dtype="uint32") = packed_params[76]
gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale4: R.Tensor((2048, 16), dtype="float16") = packed_params[77]
gpt_neox_layers_4_mlp_dense_h_to_4h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[78]
gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight4: R.Tensor((512, 256), dtype="uint32") = packed_params[79]
gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale4: R.Tensor((512, 64), dtype="float16") = packed_params[80]
gpt_neox_layers_4_mlp_dense_4h_to_h_bias4: R.Tensor((512,), dtype="float32") = packed_params[81]
gpt_neox_layers_5_input_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[82]
gpt_neox_layers_5_input_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[83]
gpt_neox_layers_5_post_attention_layernorm_weight4: R.Tensor((512,), dtype="float16") = packed_params[84]
gpt_neox_layers_5_post_attention_layernorm_bias4: R.Tensor((512,), dtype="float16") = packed_params[85]
gpt_neox_layers_5_attention_query_key_value_q_weight4: R.Tensor((1536, 64), dtype="uint32") = packed_params[86]
gpt_neox_layers_5_attention_query_key_value_q_scale4: R.Tensor((1536, 16), dtype="float16") = packed_params[87]
gpt_neox_layers_5_attention_query_key_value_bias4: R.Tensor((1536,), dtype="float16") = packed_params[88]
gpt_neox_layers_5_attention_dense_q_weight4: R.Tensor((512, 64), dtype="uint32") = packed_params[89]
gpt_neox_layers_5_attention_dense_q_scale4: R.Tensor((512, 16), dtype="float16") = packed_params[90]
gpt_neox_layers_5_attention_dense_bias4: R.Tensor((512,), dtype="float16") = packed_params[91]
gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight4: R.Tensor((2048, 64), dtype="uint32") = packed_params[92]
gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale4: R.Tensor((2048, 16), dtype="float16") = packed_params[93]
gpt_neox_layers_5_mlp_dense_h_to_4h_bias4: R.Tensor((2048,), dtype="float32") = packed_params[94]
gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight4: R.Tensor((512, 256), dtype="uint32") = packed_params[95]
gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale4: R.Tensor((512, 64), dtype="float16") = packed_params[96]
gpt_neox_layers_5_mlp_dense_4h_to_h_bias4: R.Tensor((512,), dtype="float32") = packed_params[97]
gpt_neox_final_layer_norm_weight4: R.Tensor((512,), dtype="float16") = packed_params[98]
gpt_neox_final_layer_norm_bias4: R.Tensor((512,), dtype="float16") = packed_params[99]
embed_out_q_weight4: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[100]
embed_out_q_scale4: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[101]
layer_norm39: R.Tensor((batch_size, 1, 512), dtype="float16") = R.nn.layer_norm(input_embeds, gpt_neox_layers_0_input_layernorm_weight4, gpt_neox_layers_0_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv95 = R.call_tir(cls.dequantize1, (gpt_neox_layers_0_attention_query_key_value_q_weight4, gpt_neox_layers_0_attention_query_key_value_q_scale4), out_sinfo=R.Tensor((1536, 512), dtype="float16"))
permute_dims75: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv95, axes=None)
matmul75: R.Tensor((batch_size, 1, 1536), dtype="float16") = R.matmul(layer_norm39, permute_dims75, out_dtype="void")
add108: R.Tensor((batch_size, 1, 1536), dtype="float16") = R.add(matmul75, gpt_neox_layers_0_attention_query_key_value_bias4)
reshape72: R.Tensor((batch_size, 1, 24, 64), dtype="float16") = R.reshape(add108, R.shape([batch_size, 1, 24, 64]))
reshape73: R.Tensor((batch_size, 24, 64), dtype="float16") = R.reshape(reshape72, R.shape([batch_size, 24, 64]))
lv96 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape73), out_sinfo=R.Tensor((batch_size, 8, 64), dtype="float16"))
reshape74: R.Tensor((batch_size, 1, 8, 64), dtype="float16") = R.reshape(lv96, R.shape([batch_size, 1, 8, 64]))
reshape75: R.Tensor((batch_size, 1, 512), dtype="float16") = R.reshape(reshape74, R.shape([batch_size, 1, 512]))
lv97 = R.call_tir(cls.dequantize2, (gpt_neox_layers_0_attention_dense_q_weight4, gpt_neox_layers_0_attention_dense_q_scale4), out_sinfo=R.Tensor((512, 512), dtype="float16"))
permute_dims76: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv97, axes=None)
matmul76: R.Tensor((batch_size, 1, 512), dtype="float16") = R.matmul(reshape75, permute_dims76, out_dtype="void")
add109: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(matmul76, gpt_neox_layers_0_attention_dense_bias4)
layer_norm40: R.Tensor((batch_size, 1, 512), dtype="float16") = R.nn.layer_norm(input_embeds, gpt_neox_layers_0_post_attention_layernorm_weight4, gpt_neox_layers_0_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv98 = R.call_tir(cls.dequantize3, (gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4), out_sinfo=R.Tensor((2048, 512), dtype="float16"))
permute_dims77: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv98, axes=None)
matmul77: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.matmul(layer_norm40, permute_dims77, out_dtype="float32")
add110: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(matmul77, gpt_neox_layers_0_mlp_dense_h_to_4h_bias4)
gelu18: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.nn.gelu(add110)
astype39: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(gelu18, dtype="float16")
lv99 = R.call_tir(cls.dequantize4, (gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4), out_sinfo=R.Tensor((512, 2048), dtype="float16"))
permute_dims78: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv99, axes=None)
matmul78: R.Tensor((batch_size, 1, 512), dtype="float32") = R.matmul(astype39, permute_dims78, out_dtype="float32")
add111: R.Tensor((batch_size, 1, 512), dtype="float32") = R.add(matmul78, gpt_neox_layers_0_mlp_dense_4h_to_h_bias4)
astype40: R.Tensor((batch_size, 1, 512), dtype="float16") = R.astype(add111, dtype="float16")
add112: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(astype40, add109)
add113: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(add112, input_embeds)
layer_norm41: R.Tensor((batch_size, 1, 512), dtype="float16") = R.nn.layer_norm(add113, gpt_neox_layers_1_input_layernorm_weight4, gpt_neox_layers_1_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv100 = R.call_tir(cls.dequantize1, (gpt_neox_layers_1_attention_query_key_value_q_weight4, gpt_neox_layers_1_attention_query_key_value_q_scale4), out_sinfo=R.Tensor((1536, 512), dtype="float16"))
permute_dims79: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv100, axes=None)
matmul79: R.Tensor((batch_size, 1, 1536), dtype="float16") = R.matmul(layer_norm41, permute_dims79, out_dtype="void")
add114: R.Tensor((batch_size, 1, 1536), dtype="float16") = R.add(matmul79, gpt_neox_layers_1_attention_query_key_value_bias4)
reshape76: R.Tensor((batch_size, 1, 24, 64), dtype="float16") = R.reshape(add114, R.shape([batch_size, 1, 24, 64]))
reshape77: R.Tensor((batch_size, 24, 64), dtype="float16") = R.reshape(reshape76, R.shape([batch_size, 24, 64]))
lv101 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape77), out_sinfo=R.Tensor((batch_size, 8, 64), dtype="float16"))
reshape78: R.Tensor((batch_size, 1, 8, 64), dtype="float16") = R.reshape(lv101, R.shape([batch_size, 1, 8, 64]))
reshape79: R.Tensor((batch_size, 1, 512), dtype="float16") = R.reshape(reshape78, R.shape([batch_size, 1, 512]))
lv102 = R.call_tir(cls.dequantize2, (gpt_neox_layers_1_attention_dense_q_weight4, gpt_neox_layers_1_attention_dense_q_scale4), out_sinfo=R.Tensor((512, 512), dtype="float16"))
permute_dims80: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv102, axes=None)
matmul80: R.Tensor((batch_size, 1, 512), dtype="float16") = R.matmul(reshape79, permute_dims80, out_dtype="void")
add115: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(matmul80, gpt_neox_layers_1_attention_dense_bias4)
layer_norm42: R.Tensor((batch_size, 1, 512), dtype="float16") = R.nn.layer_norm(add113, gpt_neox_layers_1_post_attention_layernorm_weight4, gpt_neox_layers_1_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv103 = R.call_tir(cls.dequantize3, (gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale4), out_sinfo=R.Tensor((2048, 512), dtype="float16"))
permute_dims81: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv103, axes=None)
matmul81: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.matmul(layer_norm42, permute_dims81, out_dtype="float32")
add116: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(matmul81, gpt_neox_layers_1_mlp_dense_h_to_4h_bias4)
gelu19: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.nn.gelu(add116)
astype41: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(gelu19, dtype="float16")
lv104 = R.call_tir(cls.dequantize4, (gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale4), out_sinfo=R.Tensor((512, 2048), dtype="float16"))
permute_dims82: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv104, axes=None)
matmul82: R.Tensor((batch_size, 1, 512), dtype="float32") = R.matmul(astype41, permute_dims82, out_dtype="float32")
add117: R.Tensor((batch_size, 1, 512), dtype="float32") = R.add(matmul82, gpt_neox_layers_1_mlp_dense_4h_to_h_bias4)
astype42: R.Tensor((batch_size, 1, 512), dtype="float16") = R.astype(add117, dtype="float16")
add118: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(astype42, add115)
add119: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(add118, add113)
layer_norm43: R.Tensor((batch_size, 1, 512), dtype="float16") = R.nn.layer_norm(add119, gpt_neox_layers_2_input_layernorm_weight4, gpt_neox_layers_2_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv105 = R.call_tir(cls.dequantize1, (gpt_neox_layers_2_attention_query_key_value_q_weight4, gpt_neox_layers_2_attention_query_key_value_q_scale4), out_sinfo=R.Tensor((1536, 512), dtype="float16"))
permute_dims83: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv105, axes=None)
matmul83: R.Tensor((batch_size, 1, 1536), dtype="float16") = R.matmul(layer_norm43, permute_dims83, out_dtype="void")
add120: R.Tensor((batch_size, 1, 1536), dtype="float16") = R.add(matmul83, gpt_neox_layers_2_attention_query_key_value_bias4)
reshape80: R.Tensor((batch_size, 1, 24, 64), dtype="float16") = R.reshape(add120, R.shape([batch_size, 1, 24, 64]))
reshape81: R.Tensor((batch_size, 24, 64), dtype="float16") = R.reshape(reshape80, R.shape([batch_size, 24, 64]))
lv106 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape81), out_sinfo=R.Tensor((batch_size, 8, 64), dtype="float16"))
reshape82: R.Tensor((batch_size, 1, 8, 64), dtype="float16") = R.reshape(lv106, R.shape([batch_size, 1, 8, 64]))
reshape83: R.Tensor((batch_size, 1, 512), dtype="float16") = R.reshape(reshape82, R.shape([batch_size, 1, 512]))
lv107 = R.call_tir(cls.dequantize2, (gpt_neox_layers_2_attention_dense_q_weight4, gpt_neox_layers_2_attention_dense_q_scale4), out_sinfo=R.Tensor((512, 512), dtype="float16"))
permute_dims84: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv107, axes=None)
matmul84: R.Tensor((batch_size, 1, 512), dtype="float16") = R.matmul(reshape83, permute_dims84, out_dtype="void")
add121: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(matmul84, gpt_neox_layers_2_attention_dense_bias4)
layer_norm44: R.Tensor((batch_size, 1, 512), dtype="float16") = R.nn.layer_norm(add119, gpt_neox_layers_2_post_attention_layernorm_weight4, gpt_neox_layers_2_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv108 = R.call_tir(cls.dequantize3, (gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale4), out_sinfo=R.Tensor((2048, 512), dtype="float16"))
permute_dims85: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv108, axes=None)
matmul85: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.matmul(layer_norm44, permute_dims85, out_dtype="float32")
add122: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(matmul85, gpt_neox_layers_2_mlp_dense_h_to_4h_bias4)
gelu20: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.nn.gelu(add122)
astype43: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(gelu20, dtype="float16")
lv109 = R.call_tir(cls.dequantize4, (gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale4), out_sinfo=R.Tensor((512, 2048), dtype="float16"))
permute_dims86: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv109, axes=None)
matmul86: R.Tensor((batch_size, 1, 512), dtype="float32") = R.matmul(astype43, permute_dims86, out_dtype="float32")
add123: R.Tensor((batch_size, 1, 512), dtype="float32") = R.add(matmul86, gpt_neox_layers_2_mlp_dense_4h_to_h_bias4)
astype44: R.Tensor((batch_size, 1, 512), dtype="float16") = R.astype(add123, dtype="float16")
add124: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(astype44, add121)
add125: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(add124, add119)
layer_norm45: R.Tensor((batch_size, 1, 512), dtype="float16") = R.nn.layer_norm(add125, gpt_neox_layers_3_input_layernorm_weight4, gpt_neox_layers_3_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv110 = R.call_tir(cls.dequantize1, (gpt_neox_layers_3_attention_query_key_value_q_weight4, gpt_neox_layers_3_attention_query_key_value_q_scale4), out_sinfo=R.Tensor((1536, 512), dtype="float16"))
permute_dims87: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv110, axes=None)
matmul87: R.Tensor((batch_size, 1, 1536), dtype="float16") = R.matmul(layer_norm45, permute_dims87, out_dtype="void")
add126: R.Tensor((batch_size, 1, 1536), dtype="float16") = R.add(matmul87, gpt_neox_layers_3_attention_query_key_value_bias4)
reshape84: R.Tensor((batch_size, 1, 24, 64), dtype="float16") = R.reshape(add126, R.shape([batch_size, 1, 24, 64]))
reshape85: R.Tensor((batch_size, 24, 64), dtype="float16") = R.reshape(reshape84, R.shape([batch_size, 24, 64]))
lv111 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape85), out_sinfo=R.Tensor((batch_size, 8, 64), dtype="float16"))
reshape86: R.Tensor((batch_size, 1, 8, 64), dtype="float16") = R.reshape(lv111, R.shape([batch_size, 1, 8, 64]))
reshape87: R.Tensor((batch_size, 1, 512), dtype="float16") = R.reshape(reshape86, R.shape([batch_size, 1, 512]))
lv112 = R.call_tir(cls.dequantize2, (gpt_neox_layers_3_attention_dense_q_weight4, gpt_neox_layers_3_attention_dense_q_scale4), out_sinfo=R.Tensor((512, 512), dtype="float16"))
permute_dims88: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv112, axes=None)
matmul88: R.Tensor((batch_size, 1, 512), dtype="float16") = R.matmul(reshape87, permute_dims88, out_dtype="void")
add127: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(matmul88, gpt_neox_layers_3_attention_dense_bias4)
layer_norm46: R.Tensor((batch_size, 1, 512), dtype="float16") = R.nn.layer_norm(add125, gpt_neox_layers_3_post_attention_layernorm_weight4, gpt_neox_layers_3_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv113 = R.call_tir(cls.dequantize3, (gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale4), out_sinfo=R.Tensor((2048, 512), dtype="float16"))
permute_dims89: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv113, axes=None)
matmul89: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.matmul(layer_norm46, permute_dims89, out_dtype="float32")
add128: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(matmul89, gpt_neox_layers_3_mlp_dense_h_to_4h_bias4)
gelu21: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.nn.gelu(add128)
astype45: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(gelu21, dtype="float16")
lv114 = R.call_tir(cls.dequantize4, (gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale4), out_sinfo=R.Tensor((512, 2048), dtype="float16"))
permute_dims90: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv114, axes=None)
matmul90: R.Tensor((batch_size, 1, 512), dtype="float32") = R.matmul(astype45, permute_dims90, out_dtype="float32")
add129: R.Tensor((batch_size, 1, 512), dtype="float32") = R.add(matmul90, gpt_neox_layers_3_mlp_dense_4h_to_h_bias4)
astype46: R.Tensor((batch_size, 1, 512), dtype="float16") = R.astype(add129, dtype="float16")
add130: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(astype46, add127)
add131: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(add130, add125)
layer_norm47: R.Tensor((batch_size, 1, 512), dtype="float16") = R.nn.layer_norm(add131, gpt_neox_layers_4_input_layernorm_weight4, gpt_neox_layers_4_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv115 = R.call_tir(cls.dequantize1, (gpt_neox_layers_4_attention_query_key_value_q_weight4, gpt_neox_layers_4_attention_query_key_value_q_scale4), out_sinfo=R.Tensor((1536, 512), dtype="float16"))
permute_dims91: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv115, axes=None)
matmul91: R.Tensor((batch_size, 1, 1536), dtype="float16") = R.matmul(layer_norm47, permute_dims91, out_dtype="void")
add132: R.Tensor((batch_size, 1, 1536), dtype="float16") = R.add(matmul91, gpt_neox_layers_4_attention_query_key_value_bias4)
reshape88: R.Tensor((batch_size, 1, 24, 64), dtype="float16") = R.reshape(add132, R.shape([batch_size, 1, 24, 64]))
reshape89: R.Tensor((batch_size, 24, 64), dtype="float16") = R.reshape(reshape88, R.shape([batch_size, 24, 64]))
lv116 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape89), out_sinfo=R.Tensor((batch_size, 8, 64), dtype="float16"))
reshape90: R.Tensor((batch_size, 1, 8, 64), dtype="float16") = R.reshape(lv116, R.shape([batch_size, 1, 8, 64]))
reshape91: R.Tensor((batch_size, 1, 512), dtype="float16") = R.reshape(reshape90, R.shape([batch_size, 1, 512]))
lv117 = R.call_tir(cls.dequantize2, (gpt_neox_layers_4_attention_dense_q_weight4, gpt_neox_layers_4_attention_dense_q_scale4), out_sinfo=R.Tensor((512, 512), dtype="float16"))
permute_dims92: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv117, axes=None)
matmul92: R.Tensor((batch_size, 1, 512), dtype="float16") = R.matmul(reshape91, permute_dims92, out_dtype="void")
add133: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(matmul92, gpt_neox_layers_4_attention_dense_bias4)
layer_norm48: R.Tensor((batch_size, 1, 512), dtype="float16") = R.nn.layer_norm(add131, gpt_neox_layers_4_post_attention_layernorm_weight4, gpt_neox_layers_4_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv118 = R.call_tir(cls.dequantize3, (gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale4), out_sinfo=R.Tensor((2048, 512), dtype="float16"))
permute_dims93: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv118, axes=None)
matmul93: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.matmul(layer_norm48, permute_dims93, out_dtype="float32")
add134: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(matmul93, gpt_neox_layers_4_mlp_dense_h_to_4h_bias4)
gelu22: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.nn.gelu(add134)
astype47: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(gelu22, dtype="float16")
lv119 = R.call_tir(cls.dequantize4, (gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale4), out_sinfo=R.Tensor((512, 2048), dtype="float16"))
permute_dims94: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv119, axes=None)
matmul94: R.Tensor((batch_size, 1, 512), dtype="float32") = R.matmul(astype47, permute_dims94, out_dtype="float32")
add135: R.Tensor((batch_size, 1, 512), dtype="float32") = R.add(matmul94, gpt_neox_layers_4_mlp_dense_4h_to_h_bias4)
astype48: R.Tensor((batch_size, 1, 512), dtype="float16") = R.astype(add135, dtype="float16")
add136: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(astype48, add133)
add137: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(add136, add131)
layer_norm49: R.Tensor((batch_size, 1, 512), dtype="float16") = R.nn.layer_norm(add137, gpt_neox_layers_5_input_layernorm_weight4, gpt_neox_layers_5_input_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv120 = R.call_tir(cls.dequantize1, (gpt_neox_layers_5_attention_query_key_value_q_weight4, gpt_neox_layers_5_attention_query_key_value_q_scale4), out_sinfo=R.Tensor((1536, 512), dtype="float16"))
permute_dims95: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv120, axes=None)
matmul95: R.Tensor((batch_size, 1, 1536), dtype="float16") = R.matmul(layer_norm49, permute_dims95, out_dtype="void")
add138: R.Tensor((batch_size, 1, 1536), dtype="float16") = R.add(matmul95, gpt_neox_layers_5_attention_query_key_value_bias4)
reshape92: R.Tensor((batch_size, 1, 24, 64), dtype="float16") = R.reshape(add138, R.shape([batch_size, 1, 24, 64]))
reshape93: R.Tensor((batch_size, 24, 64), dtype="float16") = R.reshape(reshape92, R.shape([batch_size, 24, 64]))
lv121 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape93), out_sinfo=R.Tensor((batch_size, 8, 64), dtype="float16"))
reshape94: R.Tensor((batch_size, 1, 8, 64), dtype="float16") = R.reshape(lv121, R.shape([batch_size, 1, 8, 64]))
reshape95: R.Tensor((batch_size, 1, 512), dtype="float16") = R.reshape(reshape94, R.shape([batch_size, 1, 512]))
lv122 = R.call_tir(cls.dequantize2, (gpt_neox_layers_5_attention_dense_q_weight4, gpt_neox_layers_5_attention_dense_q_scale4), out_sinfo=R.Tensor((512, 512), dtype="float16"))
permute_dims96: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv122, axes=None)
matmul96: R.Tensor((batch_size, 1, 512), dtype="float16") = R.matmul(reshape95, permute_dims96, out_dtype="void")
add139: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(matmul96, gpt_neox_layers_5_attention_dense_bias4)
layer_norm50: R.Tensor((batch_size, 1, 512), dtype="float16") = R.nn.layer_norm(add137, gpt_neox_layers_5_post_attention_layernorm_weight4, gpt_neox_layers_5_post_attention_layernorm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv123 = R.call_tir(cls.dequantize3, (gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight4, gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale4), out_sinfo=R.Tensor((2048, 512), dtype="float16"))
permute_dims97: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv123, axes=None)
matmul97: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.matmul(layer_norm50, permute_dims97, out_dtype="float32")
add140: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.add(matmul97, gpt_neox_layers_5_mlp_dense_h_to_4h_bias4)
gelu23: R.Tensor((batch_size, 1, 2048), dtype="float32") = R.nn.gelu(add140)
astype49: R.Tensor((batch_size, 1, 2048), dtype="float16") = R.astype(gelu23, dtype="float16")
lv124 = R.call_tir(cls.dequantize4, (gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight4, gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale4), out_sinfo=R.Tensor((512, 2048), dtype="float16"))
permute_dims98: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv124, axes=None)
matmul98: R.Tensor((batch_size, 1, 512), dtype="float32") = R.matmul(astype49, permute_dims98, out_dtype="float32")
add141: R.Tensor((batch_size, 1, 512), dtype="float32") = R.add(matmul98, gpt_neox_layers_5_mlp_dense_4h_to_h_bias4)
astype50: R.Tensor((batch_size, 1, 512), dtype="float16") = R.astype(add141, dtype="float16")
add142: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(astype50, add139)
add143: R.Tensor((batch_size, 1, 512), dtype="float16") = R.add(add142, add137)
layer_norm51: R.Tensor((batch_size, 1, 512), dtype="float16") = R.nn.layer_norm(add143, gpt_neox_final_layer_norm_weight4, gpt_neox_final_layer_norm_bias4, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv125 = R.call_tir(cls.dequantize, (embed_out_q_weight4, embed_out_q_scale4), out_sinfo=R.Tensor((vocab_size, 512), dtype="float16"))
permute_dims99: R.Tensor((512, vocab_size), dtype="float16") = R.permute_dims(lv125, axes=None)
matmul99: R.Tensor((batch_size, 1, vocab_size), dtype="float16") = R.matmul(layer_norm51, permute_dims99, out_dtype="void")
astype51: R.Tensor((batch_size, 1, vocab_size), dtype="float32") = R.astype(matmul99, dtype="float32")
gv4: R.Tuple(R.Tensor((batch_size, 1, vocab_size), dtype="float32"), R.Object) = astype51, paged_kv_cache
R.output(gv4)
return gv4
@R.function
def batch_prefill(input_embeds: R.Tensor((1, "seq_len", 512), dtype="float16"), logit_positions: R.Tensor(("batch_size",), dtype="int32"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"))) -> R.Tuple(R.Tensor((1, "batch_size", "vocab_size"), dtype="float32"), R.Object):
batch_size = T.int64()
vocab_size = T.int64()
seq_len = T.int64()
R.func_attr({"num_input": 3, "pipeline_parallel_stages": 1, "relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}})
cls = Module
with R.dataflow():
gpt_neox_embed_in_q_weight3: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[0]
gpt_neox_embed_in_q_scale3: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[1]
gpt_neox_layers_0_input_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[2]
gpt_neox_layers_0_input_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[3]
gpt_neox_layers_0_post_attention_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[4]
gpt_neox_layers_0_post_attention_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[5]
gpt_neox_layers_0_attention_query_key_value_q_weight3: R.Tensor((1536, 64), dtype="uint32") = packed_params[6]
gpt_neox_layers_0_attention_query_key_value_q_scale3: R.Tensor((1536, 16), dtype="float16") = packed_params[7]
gpt_neox_layers_0_attention_query_key_value_bias3: R.Tensor((1536,), dtype="float16") = packed_params[8]
gpt_neox_layers_0_attention_dense_q_weight3: R.Tensor((512, 64), dtype="uint32") = packed_params[9]
gpt_neox_layers_0_attention_dense_q_scale3: R.Tensor((512, 16), dtype="float16") = packed_params[10]
gpt_neox_layers_0_attention_dense_bias3: R.Tensor((512,), dtype="float16") = packed_params[11]
gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3: R.Tensor((2048, 64), dtype="uint32") = packed_params[12]
gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3: R.Tensor((2048, 16), dtype="float16") = packed_params[13]
gpt_neox_layers_0_mlp_dense_h_to_4h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[14]
gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3: R.Tensor((512, 256), dtype="uint32") = packed_params[15]
gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3: R.Tensor((512, 64), dtype="float16") = packed_params[16]
gpt_neox_layers_0_mlp_dense_4h_to_h_bias3: R.Tensor((512,), dtype="float32") = packed_params[17]
gpt_neox_layers_1_input_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[18]
gpt_neox_layers_1_input_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[19]
gpt_neox_layers_1_post_attention_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[20]
gpt_neox_layers_1_post_attention_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[21]
gpt_neox_layers_1_attention_query_key_value_q_weight3: R.Tensor((1536, 64), dtype="uint32") = packed_params[22]
gpt_neox_layers_1_attention_query_key_value_q_scale3: R.Tensor((1536, 16), dtype="float16") = packed_params[23]
gpt_neox_layers_1_attention_query_key_value_bias3: R.Tensor((1536,), dtype="float16") = packed_params[24]
gpt_neox_layers_1_attention_dense_q_weight3: R.Tensor((512, 64), dtype="uint32") = packed_params[25]
gpt_neox_layers_1_attention_dense_q_scale3: R.Tensor((512, 16), dtype="float16") = packed_params[26]
gpt_neox_layers_1_attention_dense_bias3: R.Tensor((512,), dtype="float16") = packed_params[27]
gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight3: R.Tensor((2048, 64), dtype="uint32") = packed_params[28]
gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale3: R.Tensor((2048, 16), dtype="float16") = packed_params[29]
gpt_neox_layers_1_mlp_dense_h_to_4h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[30]
gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight3: R.Tensor((512, 256), dtype="uint32") = packed_params[31]
gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale3: R.Tensor((512, 64), dtype="float16") = packed_params[32]
gpt_neox_layers_1_mlp_dense_4h_to_h_bias3: R.Tensor((512,), dtype="float32") = packed_params[33]
gpt_neox_layers_2_input_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[34]
gpt_neox_layers_2_input_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[35]
gpt_neox_layers_2_post_attention_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[36]
gpt_neox_layers_2_post_attention_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[37]
gpt_neox_layers_2_attention_query_key_value_q_weight3: R.Tensor((1536, 64), dtype="uint32") = packed_params[38]
gpt_neox_layers_2_attention_query_key_value_q_scale3: R.Tensor((1536, 16), dtype="float16") = packed_params[39]
gpt_neox_layers_2_attention_query_key_value_bias3: R.Tensor((1536,), dtype="float16") = packed_params[40]
gpt_neox_layers_2_attention_dense_q_weight3: R.Tensor((512, 64), dtype="uint32") = packed_params[41]
gpt_neox_layers_2_attention_dense_q_scale3: R.Tensor((512, 16), dtype="float16") = packed_params[42]
gpt_neox_layers_2_attention_dense_bias3: R.Tensor((512,), dtype="float16") = packed_params[43]
gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight3: R.Tensor((2048, 64), dtype="uint32") = packed_params[44]
gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale3: R.Tensor((2048, 16), dtype="float16") = packed_params[45]
gpt_neox_layers_2_mlp_dense_h_to_4h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[46]
gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight3: R.Tensor((512, 256), dtype="uint32") = packed_params[47]
gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale3: R.Tensor((512, 64), dtype="float16") = packed_params[48]
gpt_neox_layers_2_mlp_dense_4h_to_h_bias3: R.Tensor((512,), dtype="float32") = packed_params[49]
gpt_neox_layers_3_input_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[50]
gpt_neox_layers_3_input_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[51]
gpt_neox_layers_3_post_attention_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[52]
gpt_neox_layers_3_post_attention_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[53]
gpt_neox_layers_3_attention_query_key_value_q_weight3: R.Tensor((1536, 64), dtype="uint32") = packed_params[54]
gpt_neox_layers_3_attention_query_key_value_q_scale3: R.Tensor((1536, 16), dtype="float16") = packed_params[55]
gpt_neox_layers_3_attention_query_key_value_bias3: R.Tensor((1536,), dtype="float16") = packed_params[56]
gpt_neox_layers_3_attention_dense_q_weight3: R.Tensor((512, 64), dtype="uint32") = packed_params[57]
gpt_neox_layers_3_attention_dense_q_scale3: R.Tensor((512, 16), dtype="float16") = packed_params[58]
gpt_neox_layers_3_attention_dense_bias3: R.Tensor((512,), dtype="float16") = packed_params[59]
gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight3: R.Tensor((2048, 64), dtype="uint32") = packed_params[60]
gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale3: R.Tensor((2048, 16), dtype="float16") = packed_params[61]
gpt_neox_layers_3_mlp_dense_h_to_4h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[62]
gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight3: R.Tensor((512, 256), dtype="uint32") = packed_params[63]
gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale3: R.Tensor((512, 64), dtype="float16") = packed_params[64]
gpt_neox_layers_3_mlp_dense_4h_to_h_bias3: R.Tensor((512,), dtype="float32") = packed_params[65]
gpt_neox_layers_4_input_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[66]
gpt_neox_layers_4_input_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[67]
gpt_neox_layers_4_post_attention_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[68]
gpt_neox_layers_4_post_attention_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[69]
gpt_neox_layers_4_attention_query_key_value_q_weight3: R.Tensor((1536, 64), dtype="uint32") = packed_params[70]
gpt_neox_layers_4_attention_query_key_value_q_scale3: R.Tensor((1536, 16), dtype="float16") = packed_params[71]
gpt_neox_layers_4_attention_query_key_value_bias3: R.Tensor((1536,), dtype="float16") = packed_params[72]
gpt_neox_layers_4_attention_dense_q_weight3: R.Tensor((512, 64), dtype="uint32") = packed_params[73]
gpt_neox_layers_4_attention_dense_q_scale3: R.Tensor((512, 16), dtype="float16") = packed_params[74]
gpt_neox_layers_4_attention_dense_bias3: R.Tensor((512,), dtype="float16") = packed_params[75]
gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight3: R.Tensor((2048, 64), dtype="uint32") = packed_params[76]
gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale3: R.Tensor((2048, 16), dtype="float16") = packed_params[77]
gpt_neox_layers_4_mlp_dense_h_to_4h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[78]
gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight3: R.Tensor((512, 256), dtype="uint32") = packed_params[79]
gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale3: R.Tensor((512, 64), dtype="float16") = packed_params[80]
gpt_neox_layers_4_mlp_dense_4h_to_h_bias3: R.Tensor((512,), dtype="float32") = packed_params[81]
gpt_neox_layers_5_input_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[82]
gpt_neox_layers_5_input_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[83]
gpt_neox_layers_5_post_attention_layernorm_weight3: R.Tensor((512,), dtype="float16") = packed_params[84]
gpt_neox_layers_5_post_attention_layernorm_bias3: R.Tensor((512,), dtype="float16") = packed_params[85]
gpt_neox_layers_5_attention_query_key_value_q_weight3: R.Tensor((1536, 64), dtype="uint32") = packed_params[86]
gpt_neox_layers_5_attention_query_key_value_q_scale3: R.Tensor((1536, 16), dtype="float16") = packed_params[87]
gpt_neox_layers_5_attention_query_key_value_bias3: R.Tensor((1536,), dtype="float16") = packed_params[88]
gpt_neox_layers_5_attention_dense_q_weight3: R.Tensor((512, 64), dtype="uint32") = packed_params[89]
gpt_neox_layers_5_attention_dense_q_scale3: R.Tensor((512, 16), dtype="float16") = packed_params[90]
gpt_neox_layers_5_attention_dense_bias3: R.Tensor((512,), dtype="float16") = packed_params[91]
gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight3: R.Tensor((2048, 64), dtype="uint32") = packed_params[92]
gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale3: R.Tensor((2048, 16), dtype="float16") = packed_params[93]
gpt_neox_layers_5_mlp_dense_h_to_4h_bias3: R.Tensor((2048,), dtype="float32") = packed_params[94]
gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight3: R.Tensor((512, 256), dtype="uint32") = packed_params[95]
gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale3: R.Tensor((512, 64), dtype="float16") = packed_params[96]
gpt_neox_layers_5_mlp_dense_4h_to_h_bias3: R.Tensor((512,), dtype="float32") = packed_params[97]
gpt_neox_final_layer_norm_weight3: R.Tensor((512,), dtype="float16") = packed_params[98]
gpt_neox_final_layer_norm_bias3: R.Tensor((512,), dtype="float16") = packed_params[99]
embed_out_q_weight3: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[100]
embed_out_q_scale3: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[101]
layer_norm26: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(input_embeds, gpt_neox_layers_0_input_layernorm_weight3, gpt_neox_layers_0_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv64 = R.call_tir(cls.dequantize1, (gpt_neox_layers_0_attention_query_key_value_q_weight3, gpt_neox_layers_0_attention_query_key_value_q_scale3), out_sinfo=R.Tensor((1536, 512), dtype="float16"))
permute_dims50: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv64, axes=None)
matmul50: R.Tensor((1, seq_len, 1536), dtype="float16") = R.matmul(layer_norm26, permute_dims50, out_dtype="void")
add72: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(matmul50, gpt_neox_layers_0_attention_query_key_value_bias3)
reshape48: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add72, R.shape([1, seq_len, 24, 64]))
reshape49: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape48, R.shape([seq_len, 24, 64]))
lv65 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape49), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16"))
reshape50: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv65, R.shape([1, seq_len, 8, 64]))
reshape51: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape50, R.shape([1, seq_len, 512]))
lv66 = R.call_tir(cls.dequantize2, (gpt_neox_layers_0_attention_dense_q_weight3, gpt_neox_layers_0_attention_dense_q_scale3), out_sinfo=R.Tensor((512, 512), dtype="float16"))
permute_dims51: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv66, axes=None)
matmul51: R.Tensor((1, seq_len, 512), dtype="float16") = R.matmul(reshape51, permute_dims51, out_dtype="void")
add73: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(matmul51, gpt_neox_layers_0_attention_dense_bias3)
layer_norm27: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(input_embeds, gpt_neox_layers_0_post_attention_layernorm_weight3, gpt_neox_layers_0_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv67 = R.call_tir(cls.dequantize3, (gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3), out_sinfo=R.Tensor((2048, 512), dtype="float16"))
permute_dims52: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv67, axes=None)
matmul52: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(layer_norm27, permute_dims52, out_dtype="float32")
add74: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul52, gpt_neox_layers_0_mlp_dense_h_to_4h_bias3)
gelu12: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add74)
astype26: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu12, dtype="float16")
lv68 = R.call_tir(cls.dequantize4, (gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3), out_sinfo=R.Tensor((512, 2048), dtype="float16"))
permute_dims53: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv68, axes=None)
matmul53: R.Tensor((1, seq_len, 512), dtype="float32") = R.matmul(astype26, permute_dims53, out_dtype="float32")
add75: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(matmul53, gpt_neox_layers_0_mlp_dense_4h_to_h_bias3)
astype27: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add75, dtype="float16")
add76: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype27, add73)
add77: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add76, input_embeds)
layer_norm28: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add77, gpt_neox_layers_1_input_layernorm_weight3, gpt_neox_layers_1_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv69 = R.call_tir(cls.dequantize1, (gpt_neox_layers_1_attention_query_key_value_q_weight3, gpt_neox_layers_1_attention_query_key_value_q_scale3), out_sinfo=R.Tensor((1536, 512), dtype="float16"))
permute_dims54: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv69, axes=None)
matmul54: R.Tensor((1, seq_len, 1536), dtype="float16") = R.matmul(layer_norm28, permute_dims54, out_dtype="void")
add78: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(matmul54, gpt_neox_layers_1_attention_query_key_value_bias3)
reshape52: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add78, R.shape([1, seq_len, 24, 64]))
reshape53: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape52, R.shape([seq_len, 24, 64]))
lv70 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape53), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16"))
reshape54: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv70, R.shape([1, seq_len, 8, 64]))
reshape55: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape54, R.shape([1, seq_len, 512]))
lv71 = R.call_tir(cls.dequantize2, (gpt_neox_layers_1_attention_dense_q_weight3, gpt_neox_layers_1_attention_dense_q_scale3), out_sinfo=R.Tensor((512, 512), dtype="float16"))
permute_dims55: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv71, axes=None)
matmul55: R.Tensor((1, seq_len, 512), dtype="float16") = R.matmul(reshape55, permute_dims55, out_dtype="void")
add79: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(matmul55, gpt_neox_layers_1_attention_dense_bias3)
layer_norm29: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add77, gpt_neox_layers_1_post_attention_layernorm_weight3, gpt_neox_layers_1_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv72 = R.call_tir(cls.dequantize3, (gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale3), out_sinfo=R.Tensor((2048, 512), dtype="float16"))
permute_dims56: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv72, axes=None)
matmul56: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(layer_norm29, permute_dims56, out_dtype="float32")
add80: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul56, gpt_neox_layers_1_mlp_dense_h_to_4h_bias3)
gelu13: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add80)
astype28: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu13, dtype="float16")
lv73 = R.call_tir(cls.dequantize4, (gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale3), out_sinfo=R.Tensor((512, 2048), dtype="float16"))
permute_dims57: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv73, axes=None)
matmul57: R.Tensor((1, seq_len, 512), dtype="float32") = R.matmul(astype28, permute_dims57, out_dtype="float32")
add81: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(matmul57, gpt_neox_layers_1_mlp_dense_4h_to_h_bias3)
astype29: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add81, dtype="float16")
add82: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype29, add79)
add83: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add82, add77)
layer_norm30: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add83, gpt_neox_layers_2_input_layernorm_weight3, gpt_neox_layers_2_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv74 = R.call_tir(cls.dequantize1, (gpt_neox_layers_2_attention_query_key_value_q_weight3, gpt_neox_layers_2_attention_query_key_value_q_scale3), out_sinfo=R.Tensor((1536, 512), dtype="float16"))
permute_dims58: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv74, axes=None)
matmul58: R.Tensor((1, seq_len, 1536), dtype="float16") = R.matmul(layer_norm30, permute_dims58, out_dtype="void")
add84: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(matmul58, gpt_neox_layers_2_attention_query_key_value_bias3)
reshape56: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add84, R.shape([1, seq_len, 24, 64]))
reshape57: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape56, R.shape([seq_len, 24, 64]))
lv75 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape57), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16"))
reshape58: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv75, R.shape([1, seq_len, 8, 64]))
reshape59: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape58, R.shape([1, seq_len, 512]))
lv76 = R.call_tir(cls.dequantize2, (gpt_neox_layers_2_attention_dense_q_weight3, gpt_neox_layers_2_attention_dense_q_scale3), out_sinfo=R.Tensor((512, 512), dtype="float16"))
permute_dims59: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv76, axes=None)
matmul59: R.Tensor((1, seq_len, 512), dtype="float16") = R.matmul(reshape59, permute_dims59, out_dtype="void")
add85: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(matmul59, gpt_neox_layers_2_attention_dense_bias3)
layer_norm31: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add83, gpt_neox_layers_2_post_attention_layernorm_weight3, gpt_neox_layers_2_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv77 = R.call_tir(cls.dequantize3, (gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale3), out_sinfo=R.Tensor((2048, 512), dtype="float16"))
permute_dims60: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv77, axes=None)
matmul60: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(layer_norm31, permute_dims60, out_dtype="float32")
add86: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul60, gpt_neox_layers_2_mlp_dense_h_to_4h_bias3)
gelu14: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add86)
astype30: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu14, dtype="float16")
lv78 = R.call_tir(cls.dequantize4, (gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale3), out_sinfo=R.Tensor((512, 2048), dtype="float16"))
permute_dims61: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv78, axes=None)
matmul61: R.Tensor((1, seq_len, 512), dtype="float32") = R.matmul(astype30, permute_dims61, out_dtype="float32")
add87: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(matmul61, gpt_neox_layers_2_mlp_dense_4h_to_h_bias3)
astype31: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add87, dtype="float16")
add88: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype31, add85)
add89: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add88, add83)
layer_norm32: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add89, gpt_neox_layers_3_input_layernorm_weight3, gpt_neox_layers_3_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv79 = R.call_tir(cls.dequantize1, (gpt_neox_layers_3_attention_query_key_value_q_weight3, gpt_neox_layers_3_attention_query_key_value_q_scale3), out_sinfo=R.Tensor((1536, 512), dtype="float16"))
permute_dims62: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv79, axes=None)
matmul62: R.Tensor((1, seq_len, 1536), dtype="float16") = R.matmul(layer_norm32, permute_dims62, out_dtype="void")
add90: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(matmul62, gpt_neox_layers_3_attention_query_key_value_bias3)
reshape60: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add90, R.shape([1, seq_len, 24, 64]))
reshape61: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape60, R.shape([seq_len, 24, 64]))
lv80 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape61), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16"))
reshape62: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv80, R.shape([1, seq_len, 8, 64]))
reshape63: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape62, R.shape([1, seq_len, 512]))
lv81 = R.call_tir(cls.dequantize2, (gpt_neox_layers_3_attention_dense_q_weight3, gpt_neox_layers_3_attention_dense_q_scale3), out_sinfo=R.Tensor((512, 512), dtype="float16"))
permute_dims63: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv81, axes=None)
matmul63: R.Tensor((1, seq_len, 512), dtype="float16") = R.matmul(reshape63, permute_dims63, out_dtype="void")
add91: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(matmul63, gpt_neox_layers_3_attention_dense_bias3)
layer_norm33: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add89, gpt_neox_layers_3_post_attention_layernorm_weight3, gpt_neox_layers_3_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv82 = R.call_tir(cls.dequantize3, (gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale3), out_sinfo=R.Tensor((2048, 512), dtype="float16"))
permute_dims64: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv82, axes=None)
matmul64: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(layer_norm33, permute_dims64, out_dtype="float32")
add92: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul64, gpt_neox_layers_3_mlp_dense_h_to_4h_bias3)
gelu15: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add92)
astype32: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu15, dtype="float16")
lv83 = R.call_tir(cls.dequantize4, (gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale3), out_sinfo=R.Tensor((512, 2048), dtype="float16"))
permute_dims65: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv83, axes=None)
matmul65: R.Tensor((1, seq_len, 512), dtype="float32") = R.matmul(astype32, permute_dims65, out_dtype="float32")
add93: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(matmul65, gpt_neox_layers_3_mlp_dense_4h_to_h_bias3)
astype33: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add93, dtype="float16")
add94: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype33, add91)
add95: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add94, add89)
layer_norm34: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add95, gpt_neox_layers_4_input_layernorm_weight3, gpt_neox_layers_4_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv84 = R.call_tir(cls.dequantize1, (gpt_neox_layers_4_attention_query_key_value_q_weight3, gpt_neox_layers_4_attention_query_key_value_q_scale3), out_sinfo=R.Tensor((1536, 512), dtype="float16"))
permute_dims66: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv84, axes=None)
matmul66: R.Tensor((1, seq_len, 1536), dtype="float16") = R.matmul(layer_norm34, permute_dims66, out_dtype="void")
add96: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(matmul66, gpt_neox_layers_4_attention_query_key_value_bias3)
reshape64: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add96, R.shape([1, seq_len, 24, 64]))
reshape65: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape64, R.shape([seq_len, 24, 64]))
lv85 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape65), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16"))
reshape66: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv85, R.shape([1, seq_len, 8, 64]))
reshape67: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape66, R.shape([1, seq_len, 512]))
lv86 = R.call_tir(cls.dequantize2, (gpt_neox_layers_4_attention_dense_q_weight3, gpt_neox_layers_4_attention_dense_q_scale3), out_sinfo=R.Tensor((512, 512), dtype="float16"))
permute_dims67: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv86, axes=None)
matmul67: R.Tensor((1, seq_len, 512), dtype="float16") = R.matmul(reshape67, permute_dims67, out_dtype="void")
add97: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(matmul67, gpt_neox_layers_4_attention_dense_bias3)
layer_norm35: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add95, gpt_neox_layers_4_post_attention_layernorm_weight3, gpt_neox_layers_4_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv87 = R.call_tir(cls.dequantize3, (gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale3), out_sinfo=R.Tensor((2048, 512), dtype="float16"))
permute_dims68: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv87, axes=None)
matmul68: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(layer_norm35, permute_dims68, out_dtype="float32")
add98: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul68, gpt_neox_layers_4_mlp_dense_h_to_4h_bias3)
gelu16: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add98)
astype34: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu16, dtype="float16")
lv88 = R.call_tir(cls.dequantize4, (gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale3), out_sinfo=R.Tensor((512, 2048), dtype="float16"))
permute_dims69: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv88, axes=None)
matmul69: R.Tensor((1, seq_len, 512), dtype="float32") = R.matmul(astype34, permute_dims69, out_dtype="float32")
add99: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(matmul69, gpt_neox_layers_4_mlp_dense_4h_to_h_bias3)
astype35: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add99, dtype="float16")
add100: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype35, add97)
add101: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add100, add95)
layer_norm36: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add101, gpt_neox_layers_5_input_layernorm_weight3, gpt_neox_layers_5_input_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv89 = R.call_tir(cls.dequantize1, (gpt_neox_layers_5_attention_query_key_value_q_weight3, gpt_neox_layers_5_attention_query_key_value_q_scale3), out_sinfo=R.Tensor((1536, 512), dtype="float16"))
permute_dims70: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv89, axes=None)
matmul70: R.Tensor((1, seq_len, 1536), dtype="float16") = R.matmul(layer_norm36, permute_dims70, out_dtype="void")
add102: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(matmul70, gpt_neox_layers_5_attention_query_key_value_bias3)
reshape68: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add102, R.shape([1, seq_len, 24, 64]))
reshape69: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape68, R.shape([seq_len, 24, 64]))
lv90 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape69), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16"))
reshape70: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv90, R.shape([1, seq_len, 8, 64]))
reshape71: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape70, R.shape([1, seq_len, 512]))
lv91 = R.call_tir(cls.dequantize2, (gpt_neox_layers_5_attention_dense_q_weight3, gpt_neox_layers_5_attention_dense_q_scale3), out_sinfo=R.Tensor((512, 512), dtype="float16"))
permute_dims71: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv91, axes=None)
matmul71: R.Tensor((1, seq_len, 512), dtype="float16") = R.matmul(reshape71, permute_dims71, out_dtype="void")
add103: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(matmul71, gpt_neox_layers_5_attention_dense_bias3)
layer_norm37: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add101, gpt_neox_layers_5_post_attention_layernorm_weight3, gpt_neox_layers_5_post_attention_layernorm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv92 = R.call_tir(cls.dequantize3, (gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight3, gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale3), out_sinfo=R.Tensor((2048, 512), dtype="float16"))
permute_dims72: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv92, axes=None)
matmul72: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(layer_norm37, permute_dims72, out_dtype="float32")
add104: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul72, gpt_neox_layers_5_mlp_dense_h_to_4h_bias3)
gelu17: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add104)
astype36: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu17, dtype="float16")
lv93 = R.call_tir(cls.dequantize4, (gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight3, gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale3), out_sinfo=R.Tensor((512, 2048), dtype="float16"))
permute_dims73: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv93, axes=None)
matmul73: R.Tensor((1, seq_len, 512), dtype="float32") = R.matmul(astype36, permute_dims73, out_dtype="float32")
add105: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(matmul73, gpt_neox_layers_5_mlp_dense_4h_to_h_bias3)
astype37: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add105, dtype="float16")
add106: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype37, add103)
add107: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add106, add101)
layer_norm38: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add107, gpt_neox_final_layer_norm_weight3, gpt_neox_final_layer_norm_bias3, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
take1: R.Tensor((1, batch_size, 512), dtype="float16") = R.take(layer_norm38, logit_positions, axis=1)
lv94 = R.call_tir(cls.dequantize, (embed_out_q_weight3, embed_out_q_scale3), out_sinfo=R.Tensor((vocab_size, 512), dtype="float16"))
permute_dims74: R.Tensor((512, vocab_size), dtype="float16") = R.permute_dims(lv94, axes=None)
matmul74: R.Tensor((1, batch_size, vocab_size), dtype="float16") = R.matmul(take1, permute_dims74, out_dtype="void")
astype38: R.Tensor((1, batch_size, vocab_size), dtype="float32") = R.astype(matmul74, dtype="float32")
gv3: R.Tuple(R.Tensor((1, batch_size, vocab_size), dtype="float32"), R.Object) = astype38, paged_kv_cache
R.output(gv3)
return gv3
@R.function
def batch_verify(input_embeds: R.Tensor((1, "seq_len", 512), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"))) -> R.Tuple(R.Tensor((1, "seq_len", "vocab_size"), dtype="float32"), R.Object):
seq_len = T.int64()
vocab_size = T.int64()
R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.memory_plan_dynamic_func_output": True, "relax.rewrite_cuda_graph.capture_symbolic_vars": ["batch_size", "seq_len"], "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}})
cls = Module
with R.dataflow():
gpt_neox_embed_in_q_weight5: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[0]
gpt_neox_embed_in_q_scale5: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[1]
gpt_neox_layers_0_input_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[2]
gpt_neox_layers_0_input_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[3]
gpt_neox_layers_0_post_attention_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[4]
gpt_neox_layers_0_post_attention_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[5]
gpt_neox_layers_0_attention_query_key_value_q_weight5: R.Tensor((1536, 64), dtype="uint32") = packed_params[6]
gpt_neox_layers_0_attention_query_key_value_q_scale5: R.Tensor((1536, 16), dtype="float16") = packed_params[7]
gpt_neox_layers_0_attention_query_key_value_bias5: R.Tensor((1536,), dtype="float16") = packed_params[8]
gpt_neox_layers_0_attention_dense_q_weight5: R.Tensor((512, 64), dtype="uint32") = packed_params[9]
gpt_neox_layers_0_attention_dense_q_scale5: R.Tensor((512, 16), dtype="float16") = packed_params[10]
gpt_neox_layers_0_attention_dense_bias5: R.Tensor((512,), dtype="float16") = packed_params[11]
gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight5: R.Tensor((2048, 64), dtype="uint32") = packed_params[12]
gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale5: R.Tensor((2048, 16), dtype="float16") = packed_params[13]
gpt_neox_layers_0_mlp_dense_h_to_4h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[14]
gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight5: R.Tensor((512, 256), dtype="uint32") = packed_params[15]
gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale5: R.Tensor((512, 64), dtype="float16") = packed_params[16]
gpt_neox_layers_0_mlp_dense_4h_to_h_bias5: R.Tensor((512,), dtype="float32") = packed_params[17]
gpt_neox_layers_1_input_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[18]
gpt_neox_layers_1_input_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[19]
gpt_neox_layers_1_post_attention_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[20]
gpt_neox_layers_1_post_attention_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[21]
gpt_neox_layers_1_attention_query_key_value_q_weight5: R.Tensor((1536, 64), dtype="uint32") = packed_params[22]
gpt_neox_layers_1_attention_query_key_value_q_scale5: R.Tensor((1536, 16), dtype="float16") = packed_params[23]
gpt_neox_layers_1_attention_query_key_value_bias5: R.Tensor((1536,), dtype="float16") = packed_params[24]
gpt_neox_layers_1_attention_dense_q_weight5: R.Tensor((512, 64), dtype="uint32") = packed_params[25]
gpt_neox_layers_1_attention_dense_q_scale5: R.Tensor((512, 16), dtype="float16") = packed_params[26]
gpt_neox_layers_1_attention_dense_bias5: R.Tensor((512,), dtype="float16") = packed_params[27]
gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight5: R.Tensor((2048, 64), dtype="uint32") = packed_params[28]
gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale5: R.Tensor((2048, 16), dtype="float16") = packed_params[29]
gpt_neox_layers_1_mlp_dense_h_to_4h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[30]
gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight5: R.Tensor((512, 256), dtype="uint32") = packed_params[31]
gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale5: R.Tensor((512, 64), dtype="float16") = packed_params[32]
gpt_neox_layers_1_mlp_dense_4h_to_h_bias5: R.Tensor((512,), dtype="float32") = packed_params[33]
gpt_neox_layers_2_input_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[34]
gpt_neox_layers_2_input_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[35]
gpt_neox_layers_2_post_attention_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[36]
gpt_neox_layers_2_post_attention_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[37]
gpt_neox_layers_2_attention_query_key_value_q_weight5: R.Tensor((1536, 64), dtype="uint32") = packed_params[38]
gpt_neox_layers_2_attention_query_key_value_q_scale5: R.Tensor((1536, 16), dtype="float16") = packed_params[39]
gpt_neox_layers_2_attention_query_key_value_bias5: R.Tensor((1536,), dtype="float16") = packed_params[40]
gpt_neox_layers_2_attention_dense_q_weight5: R.Tensor((512, 64), dtype="uint32") = packed_params[41]
gpt_neox_layers_2_attention_dense_q_scale5: R.Tensor((512, 16), dtype="float16") = packed_params[42]
gpt_neox_layers_2_attention_dense_bias5: R.Tensor((512,), dtype="float16") = packed_params[43]
gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight5: R.Tensor((2048, 64), dtype="uint32") = packed_params[44]
gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale5: R.Tensor((2048, 16), dtype="float16") = packed_params[45]
gpt_neox_layers_2_mlp_dense_h_to_4h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[46]
gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight5: R.Tensor((512, 256), dtype="uint32") = packed_params[47]
gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale5: R.Tensor((512, 64), dtype="float16") = packed_params[48]
gpt_neox_layers_2_mlp_dense_4h_to_h_bias5: R.Tensor((512,), dtype="float32") = packed_params[49]
gpt_neox_layers_3_input_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[50]
gpt_neox_layers_3_input_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[51]
gpt_neox_layers_3_post_attention_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[52]
gpt_neox_layers_3_post_attention_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[53]
gpt_neox_layers_3_attention_query_key_value_q_weight5: R.Tensor((1536, 64), dtype="uint32") = packed_params[54]
gpt_neox_layers_3_attention_query_key_value_q_scale5: R.Tensor((1536, 16), dtype="float16") = packed_params[55]
gpt_neox_layers_3_attention_query_key_value_bias5: R.Tensor((1536,), dtype="float16") = packed_params[56]
gpt_neox_layers_3_attention_dense_q_weight5: R.Tensor((512, 64), dtype="uint32") = packed_params[57]
gpt_neox_layers_3_attention_dense_q_scale5: R.Tensor((512, 16), dtype="float16") = packed_params[58]
gpt_neox_layers_3_attention_dense_bias5: R.Tensor((512,), dtype="float16") = packed_params[59]
gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight5: R.Tensor((2048, 64), dtype="uint32") = packed_params[60]
gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale5: R.Tensor((2048, 16), dtype="float16") = packed_params[61]
gpt_neox_layers_3_mlp_dense_h_to_4h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[62]
gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight5: R.Tensor((512, 256), dtype="uint32") = packed_params[63]
gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale5: R.Tensor((512, 64), dtype="float16") = packed_params[64]
gpt_neox_layers_3_mlp_dense_4h_to_h_bias5: R.Tensor((512,), dtype="float32") = packed_params[65]
gpt_neox_layers_4_input_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[66]
gpt_neox_layers_4_input_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[67]
gpt_neox_layers_4_post_attention_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[68]
gpt_neox_layers_4_post_attention_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[69]
gpt_neox_layers_4_attention_query_key_value_q_weight5: R.Tensor((1536, 64), dtype="uint32") = packed_params[70]
gpt_neox_layers_4_attention_query_key_value_q_scale5: R.Tensor((1536, 16), dtype="float16") = packed_params[71]
gpt_neox_layers_4_attention_query_key_value_bias5: R.Tensor((1536,), dtype="float16") = packed_params[72]
gpt_neox_layers_4_attention_dense_q_weight5: R.Tensor((512, 64), dtype="uint32") = packed_params[73]
gpt_neox_layers_4_attention_dense_q_scale5: R.Tensor((512, 16), dtype="float16") = packed_params[74]
gpt_neox_layers_4_attention_dense_bias5: R.Tensor((512,), dtype="float16") = packed_params[75]
gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight5: R.Tensor((2048, 64), dtype="uint32") = packed_params[76]
gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale5: R.Tensor((2048, 16), dtype="float16") = packed_params[77]
gpt_neox_layers_4_mlp_dense_h_to_4h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[78]
gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight5: R.Tensor((512, 256), dtype="uint32") = packed_params[79]
gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale5: R.Tensor((512, 64), dtype="float16") = packed_params[80]
gpt_neox_layers_4_mlp_dense_4h_to_h_bias5: R.Tensor((512,), dtype="float32") = packed_params[81]
gpt_neox_layers_5_input_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[82]
gpt_neox_layers_5_input_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[83]
gpt_neox_layers_5_post_attention_layernorm_weight5: R.Tensor((512,), dtype="float16") = packed_params[84]
gpt_neox_layers_5_post_attention_layernorm_bias5: R.Tensor((512,), dtype="float16") = packed_params[85]
gpt_neox_layers_5_attention_query_key_value_q_weight5: R.Tensor((1536, 64), dtype="uint32") = packed_params[86]
gpt_neox_layers_5_attention_query_key_value_q_scale5: R.Tensor((1536, 16), dtype="float16") = packed_params[87]
gpt_neox_layers_5_attention_query_key_value_bias5: R.Tensor((1536,), dtype="float16") = packed_params[88]
gpt_neox_layers_5_attention_dense_q_weight5: R.Tensor((512, 64), dtype="uint32") = packed_params[89]
gpt_neox_layers_5_attention_dense_q_scale5: R.Tensor((512, 16), dtype="float16") = packed_params[90]
gpt_neox_layers_5_attention_dense_bias5: R.Tensor((512,), dtype="float16") = packed_params[91]
gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight5: R.Tensor((2048, 64), dtype="uint32") = packed_params[92]
gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale5: R.Tensor((2048, 16), dtype="float16") = packed_params[93]
gpt_neox_layers_5_mlp_dense_h_to_4h_bias5: R.Tensor((2048,), dtype="float32") = packed_params[94]
gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight5: R.Tensor((512, 256), dtype="uint32") = packed_params[95]
gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale5: R.Tensor((512, 64), dtype="float16") = packed_params[96]
gpt_neox_layers_5_mlp_dense_4h_to_h_bias5: R.Tensor((512,), dtype="float32") = packed_params[97]
gpt_neox_final_layer_norm_weight5: R.Tensor((512,), dtype="float16") = packed_params[98]
gpt_neox_final_layer_norm_bias5: R.Tensor((512,), dtype="float16") = packed_params[99]
embed_out_q_weight5: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[100]
embed_out_q_scale5: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[101]
layer_norm52: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(input_embeds, gpt_neox_layers_0_input_layernorm_weight5, gpt_neox_layers_0_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv126 = R.call_tir(cls.dequantize1, (gpt_neox_layers_0_attention_query_key_value_q_weight5, gpt_neox_layers_0_attention_query_key_value_q_scale5), out_sinfo=R.Tensor((1536, 512), dtype="float16"))
permute_dims100: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv126, axes=None)
matmul100: R.Tensor((1, seq_len, 1536), dtype="float16") = R.matmul(layer_norm52, permute_dims100, out_dtype="void")
add144: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(matmul100, gpt_neox_layers_0_attention_query_key_value_bias5)
reshape96: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add144, R.shape([1, seq_len, 24, 64]))
reshape97: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape96, R.shape([seq_len, 24, 64]))
lv127 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape97), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16"))
reshape98: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv127, R.shape([1, seq_len, 8, 64]))
reshape99: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape98, R.shape([1, seq_len, 512]))
lv128 = R.call_tir(cls.dequantize2, (gpt_neox_layers_0_attention_dense_q_weight5, gpt_neox_layers_0_attention_dense_q_scale5), out_sinfo=R.Tensor((512, 512), dtype="float16"))
permute_dims101: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv128, axes=None)
matmul101: R.Tensor((1, seq_len, 512), dtype="float16") = R.matmul(reshape99, permute_dims101, out_dtype="void")
add145: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(matmul101, gpt_neox_layers_0_attention_dense_bias5)
layer_norm53: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(input_embeds, gpt_neox_layers_0_post_attention_layernorm_weight5, gpt_neox_layers_0_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv129 = R.call_tir(cls.dequantize3, (gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale5), out_sinfo=R.Tensor((2048, 512), dtype="float16"))
permute_dims102: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv129, axes=None)
matmul102: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(layer_norm53, permute_dims102, out_dtype="float32")
add146: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul102, gpt_neox_layers_0_mlp_dense_h_to_4h_bias5)
gelu24: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add146)
astype52: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu24, dtype="float16")
lv130 = R.call_tir(cls.dequantize4, (gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale5), out_sinfo=R.Tensor((512, 2048), dtype="float16"))
permute_dims103: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv130, axes=None)
matmul103: R.Tensor((1, seq_len, 512), dtype="float32") = R.matmul(astype52, permute_dims103, out_dtype="float32")
add147: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(matmul103, gpt_neox_layers_0_mlp_dense_4h_to_h_bias5)
astype53: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add147, dtype="float16")
add148: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype53, add145)
add149: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add148, input_embeds)
layer_norm54: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add149, gpt_neox_layers_1_input_layernorm_weight5, gpt_neox_layers_1_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv131 = R.call_tir(cls.dequantize1, (gpt_neox_layers_1_attention_query_key_value_q_weight5, gpt_neox_layers_1_attention_query_key_value_q_scale5), out_sinfo=R.Tensor((1536, 512), dtype="float16"))
permute_dims104: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv131, axes=None)
matmul104: R.Tensor((1, seq_len, 1536), dtype="float16") = R.matmul(layer_norm54, permute_dims104, out_dtype="void")
add150: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(matmul104, gpt_neox_layers_1_attention_query_key_value_bias5)
reshape100: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add150, R.shape([1, seq_len, 24, 64]))
reshape101: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape100, R.shape([seq_len, 24, 64]))
lv132 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape101), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16"))
reshape102: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv132, R.shape([1, seq_len, 8, 64]))
reshape103: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape102, R.shape([1, seq_len, 512]))
lv133 = R.call_tir(cls.dequantize2, (gpt_neox_layers_1_attention_dense_q_weight5, gpt_neox_layers_1_attention_dense_q_scale5), out_sinfo=R.Tensor((512, 512), dtype="float16"))
permute_dims105: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv133, axes=None)
matmul105: R.Tensor((1, seq_len, 512), dtype="float16") = R.matmul(reshape103, permute_dims105, out_dtype="void")
add151: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(matmul105, gpt_neox_layers_1_attention_dense_bias5)
layer_norm55: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add149, gpt_neox_layers_1_post_attention_layernorm_weight5, gpt_neox_layers_1_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv134 = R.call_tir(cls.dequantize3, (gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale5), out_sinfo=R.Tensor((2048, 512), dtype="float16"))
permute_dims106: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv134, axes=None)
matmul106: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(layer_norm55, permute_dims106, out_dtype="float32")
add152: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul106, gpt_neox_layers_1_mlp_dense_h_to_4h_bias5)
gelu25: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add152)
astype54: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu25, dtype="float16")
lv135 = R.call_tir(cls.dequantize4, (gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale5), out_sinfo=R.Tensor((512, 2048), dtype="float16"))
permute_dims107: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv135, axes=None)
matmul107: R.Tensor((1, seq_len, 512), dtype="float32") = R.matmul(astype54, permute_dims107, out_dtype="float32")
add153: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(matmul107, gpt_neox_layers_1_mlp_dense_4h_to_h_bias5)
astype55: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add153, dtype="float16")
add154: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype55, add151)
add155: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add154, add149)
layer_norm56: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add155, gpt_neox_layers_2_input_layernorm_weight5, gpt_neox_layers_2_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv136 = R.call_tir(cls.dequantize1, (gpt_neox_layers_2_attention_query_key_value_q_weight5, gpt_neox_layers_2_attention_query_key_value_q_scale5), out_sinfo=R.Tensor((1536, 512), dtype="float16"))
permute_dims108: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv136, axes=None)
matmul108: R.Tensor((1, seq_len, 1536), dtype="float16") = R.matmul(layer_norm56, permute_dims108, out_dtype="void")
add156: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(matmul108, gpt_neox_layers_2_attention_query_key_value_bias5)
reshape104: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add156, R.shape([1, seq_len, 24, 64]))
reshape105: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape104, R.shape([seq_len, 24, 64]))
lv137 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape105), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16"))
reshape106: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv137, R.shape([1, seq_len, 8, 64]))
reshape107: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape106, R.shape([1, seq_len, 512]))
lv138 = R.call_tir(cls.dequantize2, (gpt_neox_layers_2_attention_dense_q_weight5, gpt_neox_layers_2_attention_dense_q_scale5), out_sinfo=R.Tensor((512, 512), dtype="float16"))
permute_dims109: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv138, axes=None)
matmul109: R.Tensor((1, seq_len, 512), dtype="float16") = R.matmul(reshape107, permute_dims109, out_dtype="void")
add157: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(matmul109, gpt_neox_layers_2_attention_dense_bias5)
layer_norm57: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add155, gpt_neox_layers_2_post_attention_layernorm_weight5, gpt_neox_layers_2_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv139 = R.call_tir(cls.dequantize3, (gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale5), out_sinfo=R.Tensor((2048, 512), dtype="float16"))
permute_dims110: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv139, axes=None)
matmul110: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(layer_norm57, permute_dims110, out_dtype="float32")
add158: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul110, gpt_neox_layers_2_mlp_dense_h_to_4h_bias5)
gelu26: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add158)
astype56: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu26, dtype="float16")
lv140 = R.call_tir(cls.dequantize4, (gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale5), out_sinfo=R.Tensor((512, 2048), dtype="float16"))
permute_dims111: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv140, axes=None)
matmul111: R.Tensor((1, seq_len, 512), dtype="float32") = R.matmul(astype56, permute_dims111, out_dtype="float32")
add159: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(matmul111, gpt_neox_layers_2_mlp_dense_4h_to_h_bias5)
astype57: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add159, dtype="float16")
add160: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype57, add157)
add161: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add160, add155)
layer_norm58: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add161, gpt_neox_layers_3_input_layernorm_weight5, gpt_neox_layers_3_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv141 = R.call_tir(cls.dequantize1, (gpt_neox_layers_3_attention_query_key_value_q_weight5, gpt_neox_layers_3_attention_query_key_value_q_scale5), out_sinfo=R.Tensor((1536, 512), dtype="float16"))
permute_dims112: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv141, axes=None)
matmul112: R.Tensor((1, seq_len, 1536), dtype="float16") = R.matmul(layer_norm58, permute_dims112, out_dtype="void")
add162: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(matmul112, gpt_neox_layers_3_attention_query_key_value_bias5)
reshape108: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add162, R.shape([1, seq_len, 24, 64]))
reshape109: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape108, R.shape([seq_len, 24, 64]))
lv142 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape109), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16"))
reshape110: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv142, R.shape([1, seq_len, 8, 64]))
reshape111: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape110, R.shape([1, seq_len, 512]))
lv143 = R.call_tir(cls.dequantize2, (gpt_neox_layers_3_attention_dense_q_weight5, gpt_neox_layers_3_attention_dense_q_scale5), out_sinfo=R.Tensor((512, 512), dtype="float16"))
permute_dims113: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv143, axes=None)
matmul113: R.Tensor((1, seq_len, 512), dtype="float16") = R.matmul(reshape111, permute_dims113, out_dtype="void")
add163: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(matmul113, gpt_neox_layers_3_attention_dense_bias5)
layer_norm59: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add161, gpt_neox_layers_3_post_attention_layernorm_weight5, gpt_neox_layers_3_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv144 = R.call_tir(cls.dequantize3, (gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale5), out_sinfo=R.Tensor((2048, 512), dtype="float16"))
permute_dims114: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv144, axes=None)
matmul114: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(layer_norm59, permute_dims114, out_dtype="float32")
add164: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul114, gpt_neox_layers_3_mlp_dense_h_to_4h_bias5)
gelu27: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add164)
astype58: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu27, dtype="float16")
lv145 = R.call_tir(cls.dequantize4, (gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale5), out_sinfo=R.Tensor((512, 2048), dtype="float16"))
permute_dims115: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv145, axes=None)
matmul115: R.Tensor((1, seq_len, 512), dtype="float32") = R.matmul(astype58, permute_dims115, out_dtype="float32")
add165: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(matmul115, gpt_neox_layers_3_mlp_dense_4h_to_h_bias5)
astype59: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add165, dtype="float16")
add166: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype59, add163)
add167: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add166, add161)
layer_norm60: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add167, gpt_neox_layers_4_input_layernorm_weight5, gpt_neox_layers_4_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv146 = R.call_tir(cls.dequantize1, (gpt_neox_layers_4_attention_query_key_value_q_weight5, gpt_neox_layers_4_attention_query_key_value_q_scale5), out_sinfo=R.Tensor((1536, 512), dtype="float16"))
permute_dims116: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv146, axes=None)
matmul116: R.Tensor((1, seq_len, 1536), dtype="float16") = R.matmul(layer_norm60, permute_dims116, out_dtype="void")
add168: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(matmul116, gpt_neox_layers_4_attention_query_key_value_bias5)
reshape112: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add168, R.shape([1, seq_len, 24, 64]))
reshape113: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape112, R.shape([seq_len, 24, 64]))
lv147 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape113), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16"))
reshape114: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv147, R.shape([1, seq_len, 8, 64]))
reshape115: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape114, R.shape([1, seq_len, 512]))
lv148 = R.call_tir(cls.dequantize2, (gpt_neox_layers_4_attention_dense_q_weight5, gpt_neox_layers_4_attention_dense_q_scale5), out_sinfo=R.Tensor((512, 512), dtype="float16"))
permute_dims117: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv148, axes=None)
matmul117: R.Tensor((1, seq_len, 512), dtype="float16") = R.matmul(reshape115, permute_dims117, out_dtype="void")
add169: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(matmul117, gpt_neox_layers_4_attention_dense_bias5)
layer_norm61: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add167, gpt_neox_layers_4_post_attention_layernorm_weight5, gpt_neox_layers_4_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv149 = R.call_tir(cls.dequantize3, (gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale5), out_sinfo=R.Tensor((2048, 512), dtype="float16"))
permute_dims118: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv149, axes=None)
matmul118: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(layer_norm61, permute_dims118, out_dtype="float32")
add170: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul118, gpt_neox_layers_4_mlp_dense_h_to_4h_bias5)
gelu28: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add170)
astype60: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu28, dtype="float16")
lv150 = R.call_tir(cls.dequantize4, (gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale5), out_sinfo=R.Tensor((512, 2048), dtype="float16"))
permute_dims119: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv150, axes=None)
matmul119: R.Tensor((1, seq_len, 512), dtype="float32") = R.matmul(astype60, permute_dims119, out_dtype="float32")
add171: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(matmul119, gpt_neox_layers_4_mlp_dense_4h_to_h_bias5)
astype61: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add171, dtype="float16")
add172: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype61, add169)
add173: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add172, add167)
layer_norm62: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add173, gpt_neox_layers_5_input_layernorm_weight5, gpt_neox_layers_5_input_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv151 = R.call_tir(cls.dequantize1, (gpt_neox_layers_5_attention_query_key_value_q_weight5, gpt_neox_layers_5_attention_query_key_value_q_scale5), out_sinfo=R.Tensor((1536, 512), dtype="float16"))
permute_dims120: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv151, axes=None)
matmul120: R.Tensor((1, seq_len, 1536), dtype="float16") = R.matmul(layer_norm62, permute_dims120, out_dtype="void")
add174: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(matmul120, gpt_neox_layers_5_attention_query_key_value_bias5)
reshape116: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add174, R.shape([1, seq_len, 24, 64]))
reshape117: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape116, R.shape([seq_len, 24, 64]))
lv152 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape117), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16"))
reshape118: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv152, R.shape([1, seq_len, 8, 64]))
reshape119: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape118, R.shape([1, seq_len, 512]))
lv153 = R.call_tir(cls.dequantize2, (gpt_neox_layers_5_attention_dense_q_weight5, gpt_neox_layers_5_attention_dense_q_scale5), out_sinfo=R.Tensor((512, 512), dtype="float16"))
permute_dims121: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv153, axes=None)
matmul121: R.Tensor((1, seq_len, 512), dtype="float16") = R.matmul(reshape119, permute_dims121, out_dtype="void")
add175: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(matmul121, gpt_neox_layers_5_attention_dense_bias5)
layer_norm63: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add173, gpt_neox_layers_5_post_attention_layernorm_weight5, gpt_neox_layers_5_post_attention_layernorm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv154 = R.call_tir(cls.dequantize3, (gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight5, gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale5), out_sinfo=R.Tensor((2048, 512), dtype="float16"))
permute_dims122: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv154, axes=None)
matmul122: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(layer_norm63, permute_dims122, out_dtype="float32")
add176: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul122, gpt_neox_layers_5_mlp_dense_h_to_4h_bias5)
gelu29: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add176)
astype62: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu29, dtype="float16")
lv155 = R.call_tir(cls.dequantize4, (gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight5, gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale5), out_sinfo=R.Tensor((512, 2048), dtype="float16"))
permute_dims123: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv155, axes=None)
matmul123: R.Tensor((1, seq_len, 512), dtype="float32") = R.matmul(astype62, permute_dims123, out_dtype="float32")
add177: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(matmul123, gpt_neox_layers_5_mlp_dense_4h_to_h_bias5)
astype63: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add177, dtype="float16")
add178: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype63, add175)
add179: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add178, add173)
layer_norm64: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add179, gpt_neox_final_layer_norm_weight5, gpt_neox_final_layer_norm_bias5, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv156 = R.call_tir(cls.dequantize, (embed_out_q_weight5, embed_out_q_scale5), out_sinfo=R.Tensor((vocab_size, 512), dtype="float16"))
permute_dims124: R.Tensor((512, vocab_size), dtype="float16") = R.permute_dims(lv156, axes=None)
matmul124: R.Tensor((1, seq_len, vocab_size), dtype="float16") = R.matmul(layer_norm64, permute_dims124, out_dtype="void")
astype64: R.Tensor((1, seq_len, vocab_size), dtype="float32") = R.astype(matmul124, dtype="float32")
gv5: R.Tuple(R.Tensor((1, seq_len, vocab_size), dtype="float32"), R.Object) = astype64, paged_kv_cache
R.output(gv5)
return gv5
@R.function
def create_tir_paged_kv_cache(max_batch_size_: R.Shape(["max_batch_size"]), max_total_seq_len_: R.Shape(["max_total_seq_len"]), prefill_chunk_size_: R.Shape(["prefill_chunk_size"]), page_size_: R.Shape(["page_size"]), support_sliding_window_: R.Shape(["support_sliding_window"])) -> R.Object:
max_batch_size = T.int64()
max_total_seq_len = T.int64()
prefill_chunk_size = T.int64()
page_size = T.int64()
support_sliding_window = T.int64()
R.func_attr({"relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}})
cls = Module
gv: R.Tensor((), dtype="float16") = R.zeros(R.shape([]), dtype="float16")
paged_kv_cache: R.Object = R.call_pure_packed("vm.builtin.paged_attention_kv_cache_create_reduced", R.shape([max_batch_size, max_total_seq_len, prefill_chunk_size, page_size, support_sliding_window]), R.shape([0, 6]), R.prim_value(8), R.prim_value(8), R.prim_value(64), R.prim_value(1), R.prim_value(1), R.prim_value(10000), gv, cls.tir_kv_cache_transpose_append, cls.batch_prefill_paged_kv, cls.batch_decode_paged_kv, cls.batch_prefill_paged_kv_sliding_window, cls.batch_decode_paged_kv_sliding_window, cls.batch_prefill_ragged_kv, cls.merge_state_inplace, cls.fused_rope, cls.copy_single_page, cls.tir_kv_cache_debug_get_kv, cls.compact_kv_copy, cls.batch_tree_attn, cls.tree_attn_paged_kv, R.prim_value(0), R.prim_value(0), sinfo_args=(R.Object,))
return paged_kv_cache
@R.function
def decode(input_embed: R.Tensor((1, 1, 512), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"))) -> R.Tuple(R.Tensor((1, 1, "vocab_size"), dtype="float32"), R.Object):
vocab_size = T.int64()
R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}})
cls = Module
with R.dataflow():
gpt_neox_embed_in_q_weight2: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[0]
gpt_neox_embed_in_q_scale2: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[1]
gpt_neox_layers_0_input_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[2]
gpt_neox_layers_0_input_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[3]
gpt_neox_layers_0_post_attention_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[4]
gpt_neox_layers_0_post_attention_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[5]
gpt_neox_layers_0_attention_query_key_value_q_weight2: R.Tensor((1536, 64), dtype="uint32") = packed_params[6]
gpt_neox_layers_0_attention_query_key_value_q_scale2: R.Tensor((1536, 16), dtype="float16") = packed_params[7]
gpt_neox_layers_0_attention_query_key_value_bias2: R.Tensor((1536,), dtype="float16") = packed_params[8]
gpt_neox_layers_0_attention_dense_q_weight2: R.Tensor((512, 64), dtype="uint32") = packed_params[9]
gpt_neox_layers_0_attention_dense_q_scale2: R.Tensor((512, 16), dtype="float16") = packed_params[10]
gpt_neox_layers_0_attention_dense_bias2: R.Tensor((512,), dtype="float16") = packed_params[11]
gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2: R.Tensor((2048, 64), dtype="uint32") = packed_params[12]
gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2: R.Tensor((2048, 16), dtype="float16") = packed_params[13]
gpt_neox_layers_0_mlp_dense_h_to_4h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[14]
gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2: R.Tensor((512, 256), dtype="uint32") = packed_params[15]
gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2: R.Tensor((512, 64), dtype="float16") = packed_params[16]
gpt_neox_layers_0_mlp_dense_4h_to_h_bias2: R.Tensor((512,), dtype="float32") = packed_params[17]
gpt_neox_layers_1_input_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[18]
gpt_neox_layers_1_input_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[19]
gpt_neox_layers_1_post_attention_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[20]
gpt_neox_layers_1_post_attention_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[21]
gpt_neox_layers_1_attention_query_key_value_q_weight2: R.Tensor((1536, 64), dtype="uint32") = packed_params[22]
gpt_neox_layers_1_attention_query_key_value_q_scale2: R.Tensor((1536, 16), dtype="float16") = packed_params[23]
gpt_neox_layers_1_attention_query_key_value_bias2: R.Tensor((1536,), dtype="float16") = packed_params[24]
gpt_neox_layers_1_attention_dense_q_weight2: R.Tensor((512, 64), dtype="uint32") = packed_params[25]
gpt_neox_layers_1_attention_dense_q_scale2: R.Tensor((512, 16), dtype="float16") = packed_params[26]
gpt_neox_layers_1_attention_dense_bias2: R.Tensor((512,), dtype="float16") = packed_params[27]
gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight2: R.Tensor((2048, 64), dtype="uint32") = packed_params[28]
gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale2: R.Tensor((2048, 16), dtype="float16") = packed_params[29]
gpt_neox_layers_1_mlp_dense_h_to_4h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[30]
gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight2: R.Tensor((512, 256), dtype="uint32") = packed_params[31]
gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale2: R.Tensor((512, 64), dtype="float16") = packed_params[32]
gpt_neox_layers_1_mlp_dense_4h_to_h_bias2: R.Tensor((512,), dtype="float32") = packed_params[33]
gpt_neox_layers_2_input_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[34]
gpt_neox_layers_2_input_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[35]
gpt_neox_layers_2_post_attention_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[36]
gpt_neox_layers_2_post_attention_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[37]
gpt_neox_layers_2_attention_query_key_value_q_weight2: R.Tensor((1536, 64), dtype="uint32") = packed_params[38]
gpt_neox_layers_2_attention_query_key_value_q_scale2: R.Tensor((1536, 16), dtype="float16") = packed_params[39]
gpt_neox_layers_2_attention_query_key_value_bias2: R.Tensor((1536,), dtype="float16") = packed_params[40]
gpt_neox_layers_2_attention_dense_q_weight2: R.Tensor((512, 64), dtype="uint32") = packed_params[41]
gpt_neox_layers_2_attention_dense_q_scale2: R.Tensor((512, 16), dtype="float16") = packed_params[42]
gpt_neox_layers_2_attention_dense_bias2: R.Tensor((512,), dtype="float16") = packed_params[43]
gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight2: R.Tensor((2048, 64), dtype="uint32") = packed_params[44]
gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale2: R.Tensor((2048, 16), dtype="float16") = packed_params[45]
gpt_neox_layers_2_mlp_dense_h_to_4h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[46]
gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight2: R.Tensor((512, 256), dtype="uint32") = packed_params[47]
gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale2: R.Tensor((512, 64), dtype="float16") = packed_params[48]
gpt_neox_layers_2_mlp_dense_4h_to_h_bias2: R.Tensor((512,), dtype="float32") = packed_params[49]
gpt_neox_layers_3_input_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[50]
gpt_neox_layers_3_input_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[51]
gpt_neox_layers_3_post_attention_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[52]
gpt_neox_layers_3_post_attention_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[53]
gpt_neox_layers_3_attention_query_key_value_q_weight2: R.Tensor((1536, 64), dtype="uint32") = packed_params[54]
gpt_neox_layers_3_attention_query_key_value_q_scale2: R.Tensor((1536, 16), dtype="float16") = packed_params[55]
gpt_neox_layers_3_attention_query_key_value_bias2: R.Tensor((1536,), dtype="float16") = packed_params[56]
gpt_neox_layers_3_attention_dense_q_weight2: R.Tensor((512, 64), dtype="uint32") = packed_params[57]
gpt_neox_layers_3_attention_dense_q_scale2: R.Tensor((512, 16), dtype="float16") = packed_params[58]
gpt_neox_layers_3_attention_dense_bias2: R.Tensor((512,), dtype="float16") = packed_params[59]
gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight2: R.Tensor((2048, 64), dtype="uint32") = packed_params[60]
gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale2: R.Tensor((2048, 16), dtype="float16") = packed_params[61]
gpt_neox_layers_3_mlp_dense_h_to_4h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[62]
gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight2: R.Tensor((512, 256), dtype="uint32") = packed_params[63]
gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale2: R.Tensor((512, 64), dtype="float16") = packed_params[64]
gpt_neox_layers_3_mlp_dense_4h_to_h_bias2: R.Tensor((512,), dtype="float32") = packed_params[65]
gpt_neox_layers_4_input_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[66]
gpt_neox_layers_4_input_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[67]
gpt_neox_layers_4_post_attention_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[68]
gpt_neox_layers_4_post_attention_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[69]
gpt_neox_layers_4_attention_query_key_value_q_weight2: R.Tensor((1536, 64), dtype="uint32") = packed_params[70]
gpt_neox_layers_4_attention_query_key_value_q_scale2: R.Tensor((1536, 16), dtype="float16") = packed_params[71]
gpt_neox_layers_4_attention_query_key_value_bias2: R.Tensor((1536,), dtype="float16") = packed_params[72]
gpt_neox_layers_4_attention_dense_q_weight2: R.Tensor((512, 64), dtype="uint32") = packed_params[73]
gpt_neox_layers_4_attention_dense_q_scale2: R.Tensor((512, 16), dtype="float16") = packed_params[74]
gpt_neox_layers_4_attention_dense_bias2: R.Tensor((512,), dtype="float16") = packed_params[75]
gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight2: R.Tensor((2048, 64), dtype="uint32") = packed_params[76]
gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale2: R.Tensor((2048, 16), dtype="float16") = packed_params[77]
gpt_neox_layers_4_mlp_dense_h_to_4h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[78]
gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight2: R.Tensor((512, 256), dtype="uint32") = packed_params[79]
gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale2: R.Tensor((512, 64), dtype="float16") = packed_params[80]
gpt_neox_layers_4_mlp_dense_4h_to_h_bias2: R.Tensor((512,), dtype="float32") = packed_params[81]
gpt_neox_layers_5_input_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[82]
gpt_neox_layers_5_input_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[83]
gpt_neox_layers_5_post_attention_layernorm_weight2: R.Tensor((512,), dtype="float16") = packed_params[84]
gpt_neox_layers_5_post_attention_layernorm_bias2: R.Tensor((512,), dtype="float16") = packed_params[85]
gpt_neox_layers_5_attention_query_key_value_q_weight2: R.Tensor((1536, 64), dtype="uint32") = packed_params[86]
gpt_neox_layers_5_attention_query_key_value_q_scale2: R.Tensor((1536, 16), dtype="float16") = packed_params[87]
gpt_neox_layers_5_attention_query_key_value_bias2: R.Tensor((1536,), dtype="float16") = packed_params[88]
gpt_neox_layers_5_attention_dense_q_weight2: R.Tensor((512, 64), dtype="uint32") = packed_params[89]
gpt_neox_layers_5_attention_dense_q_scale2: R.Tensor((512, 16), dtype="float16") = packed_params[90]
gpt_neox_layers_5_attention_dense_bias2: R.Tensor((512,), dtype="float16") = packed_params[91]
gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight2: R.Tensor((2048, 64), dtype="uint32") = packed_params[92]
gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale2: R.Tensor((2048, 16), dtype="float16") = packed_params[93]
gpt_neox_layers_5_mlp_dense_h_to_4h_bias2: R.Tensor((2048,), dtype="float32") = packed_params[94]
gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight2: R.Tensor((512, 256), dtype="uint32") = packed_params[95]
gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale2: R.Tensor((512, 64), dtype="float16") = packed_params[96]
gpt_neox_layers_5_mlp_dense_4h_to_h_bias2: R.Tensor((512,), dtype="float32") = packed_params[97]
gpt_neox_final_layer_norm_weight2: R.Tensor((512,), dtype="float16") = packed_params[98]
gpt_neox_final_layer_norm_bias2: R.Tensor((512,), dtype="float16") = packed_params[99]
embed_out_q_weight2: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[100]
embed_out_q_scale2: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[101]
layer_norm13: R.Tensor((1, 1, 512), dtype="float16") = R.nn.layer_norm(input_embed, gpt_neox_layers_0_input_layernorm_weight2, gpt_neox_layers_0_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv33 = R.call_tir(cls.dequantize1, (gpt_neox_layers_0_attention_query_key_value_q_weight2, gpt_neox_layers_0_attention_query_key_value_q_scale2), out_sinfo=R.Tensor((1536, 512), dtype="float16"))
permute_dims25: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv33, axes=None)
matmul25: R.Tensor((1, 1, 1536), dtype="float16") = R.matmul(layer_norm13, permute_dims25, out_dtype="void")
add36: R.Tensor((1, 1, 1536), dtype="float16") = R.add(matmul25, gpt_neox_layers_0_attention_query_key_value_bias2)
reshape24: R.Tensor((1, 1, 24, 64), dtype="float16") = R.reshape(add36, R.shape([1, 1, 24, 64]))
reshape25: R.Tensor((1, 24, 64), dtype="float16") = R.reshape(reshape24, R.shape([1, 24, 64]))
lv34 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape25), out_sinfo=R.Tensor((1, 8, 64), dtype="float16"))
reshape26: R.Tensor((1, 1, 8, 64), dtype="float16") = R.reshape(lv34, R.shape([1, 1, 8, 64]))
reshape27: R.Tensor((1, 1, 512), dtype="float16") = R.reshape(reshape26, R.shape([1, 1, 512]))
lv35 = R.call_tir(cls.dequantize2, (gpt_neox_layers_0_attention_dense_q_weight2, gpt_neox_layers_0_attention_dense_q_scale2), out_sinfo=R.Tensor((512, 512), dtype="float16"))
permute_dims26: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv35, axes=None)
matmul26: R.Tensor((1, 1, 512), dtype="float16") = R.matmul(reshape27, permute_dims26, out_dtype="void")
add37: R.Tensor((1, 1, 512), dtype="float16") = R.add(matmul26, gpt_neox_layers_0_attention_dense_bias2)
layer_norm14: R.Tensor((1, 1, 512), dtype="float16") = R.nn.layer_norm(input_embed, gpt_neox_layers_0_post_attention_layernorm_weight2, gpt_neox_layers_0_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv36 = R.call_tir(cls.dequantize3, (gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2), out_sinfo=R.Tensor((2048, 512), dtype="float16"))
permute_dims27: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv36, axes=None)
matmul27: R.Tensor((1, 1, 2048), dtype="float32") = R.matmul(layer_norm14, permute_dims27, out_dtype="float32")
add38: R.Tensor((1, 1, 2048), dtype="float32") = R.add(matmul27, gpt_neox_layers_0_mlp_dense_h_to_4h_bias2)
gelu6: R.Tensor((1, 1, 2048), dtype="float32") = R.nn.gelu(add38)
astype13: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(gelu6, dtype="float16")
lv37 = R.call_tir(cls.dequantize4, (gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2), out_sinfo=R.Tensor((512, 2048), dtype="float16"))
permute_dims28: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv37, axes=None)
matmul28: R.Tensor((1, 1, 512), dtype="float32") = R.matmul(astype13, permute_dims28, out_dtype="float32")
add39: R.Tensor((1, 1, 512), dtype="float32") = R.add(matmul28, gpt_neox_layers_0_mlp_dense_4h_to_h_bias2)
astype14: R.Tensor((1, 1, 512), dtype="float16") = R.astype(add39, dtype="float16")
add40: R.Tensor((1, 1, 512), dtype="float16") = R.add(astype14, add37)
add41: R.Tensor((1, 1, 512), dtype="float16") = R.add(add40, input_embed)
layer_norm15: R.Tensor((1, 1, 512), dtype="float16") = R.nn.layer_norm(add41, gpt_neox_layers_1_input_layernorm_weight2, gpt_neox_layers_1_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv38 = R.call_tir(cls.dequantize1, (gpt_neox_layers_1_attention_query_key_value_q_weight2, gpt_neox_layers_1_attention_query_key_value_q_scale2), out_sinfo=R.Tensor((1536, 512), dtype="float16"))
permute_dims29: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv38, axes=None)
matmul29: R.Tensor((1, 1, 1536), dtype="float16") = R.matmul(layer_norm15, permute_dims29, out_dtype="void")
add42: R.Tensor((1, 1, 1536), dtype="float16") = R.add(matmul29, gpt_neox_layers_1_attention_query_key_value_bias2)
reshape28: R.Tensor((1, 1, 24, 64), dtype="float16") = R.reshape(add42, R.shape([1, 1, 24, 64]))
reshape29: R.Tensor((1, 24, 64), dtype="float16") = R.reshape(reshape28, R.shape([1, 24, 64]))
lv39 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape29), out_sinfo=R.Tensor((1, 8, 64), dtype="float16"))
reshape30: R.Tensor((1, 1, 8, 64), dtype="float16") = R.reshape(lv39, R.shape([1, 1, 8, 64]))
reshape31: R.Tensor((1, 1, 512), dtype="float16") = R.reshape(reshape30, R.shape([1, 1, 512]))
lv40 = R.call_tir(cls.dequantize2, (gpt_neox_layers_1_attention_dense_q_weight2, gpt_neox_layers_1_attention_dense_q_scale2), out_sinfo=R.Tensor((512, 512), dtype="float16"))
permute_dims30: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv40, axes=None)
matmul30: R.Tensor((1, 1, 512), dtype="float16") = R.matmul(reshape31, permute_dims30, out_dtype="void")
add43: R.Tensor((1, 1, 512), dtype="float16") = R.add(matmul30, gpt_neox_layers_1_attention_dense_bias2)
layer_norm16: R.Tensor((1, 1, 512), dtype="float16") = R.nn.layer_norm(add41, gpt_neox_layers_1_post_attention_layernorm_weight2, gpt_neox_layers_1_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv41 = R.call_tir(cls.dequantize3, (gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale2), out_sinfo=R.Tensor((2048, 512), dtype="float16"))
permute_dims31: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv41, axes=None)
matmul31: R.Tensor((1, 1, 2048), dtype="float32") = R.matmul(layer_norm16, permute_dims31, out_dtype="float32")
add44: R.Tensor((1, 1, 2048), dtype="float32") = R.add(matmul31, gpt_neox_layers_1_mlp_dense_h_to_4h_bias2)
gelu7: R.Tensor((1, 1, 2048), dtype="float32") = R.nn.gelu(add44)
astype15: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(gelu7, dtype="float16")
lv42 = R.call_tir(cls.dequantize4, (gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale2), out_sinfo=R.Tensor((512, 2048), dtype="float16"))
permute_dims32: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv42, axes=None)
matmul32: R.Tensor((1, 1, 512), dtype="float32") = R.matmul(astype15, permute_dims32, out_dtype="float32")
add45: R.Tensor((1, 1, 512), dtype="float32") = R.add(matmul32, gpt_neox_layers_1_mlp_dense_4h_to_h_bias2)
astype16: R.Tensor((1, 1, 512), dtype="float16") = R.astype(add45, dtype="float16")
add46: R.Tensor((1, 1, 512), dtype="float16") = R.add(astype16, add43)
add47: R.Tensor((1, 1, 512), dtype="float16") = R.add(add46, add41)
layer_norm17: R.Tensor((1, 1, 512), dtype="float16") = R.nn.layer_norm(add47, gpt_neox_layers_2_input_layernorm_weight2, gpt_neox_layers_2_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv43 = R.call_tir(cls.dequantize1, (gpt_neox_layers_2_attention_query_key_value_q_weight2, gpt_neox_layers_2_attention_query_key_value_q_scale2), out_sinfo=R.Tensor((1536, 512), dtype="float16"))
permute_dims33: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv43, axes=None)
matmul33: R.Tensor((1, 1, 1536), dtype="float16") = R.matmul(layer_norm17, permute_dims33, out_dtype="void")
add48: R.Tensor((1, 1, 1536), dtype="float16") = R.add(matmul33, gpt_neox_layers_2_attention_query_key_value_bias2)
reshape32: R.Tensor((1, 1, 24, 64), dtype="float16") = R.reshape(add48, R.shape([1, 1, 24, 64]))
reshape33: R.Tensor((1, 24, 64), dtype="float16") = R.reshape(reshape32, R.shape([1, 24, 64]))
lv44 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape33), out_sinfo=R.Tensor((1, 8, 64), dtype="float16"))
reshape34: R.Tensor((1, 1, 8, 64), dtype="float16") = R.reshape(lv44, R.shape([1, 1, 8, 64]))
reshape35: R.Tensor((1, 1, 512), dtype="float16") = R.reshape(reshape34, R.shape([1, 1, 512]))
lv45 = R.call_tir(cls.dequantize2, (gpt_neox_layers_2_attention_dense_q_weight2, gpt_neox_layers_2_attention_dense_q_scale2), out_sinfo=R.Tensor((512, 512), dtype="float16"))
permute_dims34: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv45, axes=None)
matmul34: R.Tensor((1, 1, 512), dtype="float16") = R.matmul(reshape35, permute_dims34, out_dtype="void")
add49: R.Tensor((1, 1, 512), dtype="float16") = R.add(matmul34, gpt_neox_layers_2_attention_dense_bias2)
layer_norm18: R.Tensor((1, 1, 512), dtype="float16") = R.nn.layer_norm(add47, gpt_neox_layers_2_post_attention_layernorm_weight2, gpt_neox_layers_2_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv46 = R.call_tir(cls.dequantize3, (gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale2), out_sinfo=R.Tensor((2048, 512), dtype="float16"))
permute_dims35: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv46, axes=None)
matmul35: R.Tensor((1, 1, 2048), dtype="float32") = R.matmul(layer_norm18, permute_dims35, out_dtype="float32")
add50: R.Tensor((1, 1, 2048), dtype="float32") = R.add(matmul35, gpt_neox_layers_2_mlp_dense_h_to_4h_bias2)
gelu8: R.Tensor((1, 1, 2048), dtype="float32") = R.nn.gelu(add50)
astype17: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(gelu8, dtype="float16")
lv47 = R.call_tir(cls.dequantize4, (gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale2), out_sinfo=R.Tensor((512, 2048), dtype="float16"))
permute_dims36: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv47, axes=None)
matmul36: R.Tensor((1, 1, 512), dtype="float32") = R.matmul(astype17, permute_dims36, out_dtype="float32")
add51: R.Tensor((1, 1, 512), dtype="float32") = R.add(matmul36, gpt_neox_layers_2_mlp_dense_4h_to_h_bias2)
astype18: R.Tensor((1, 1, 512), dtype="float16") = R.astype(add51, dtype="float16")
add52: R.Tensor((1, 1, 512), dtype="float16") = R.add(astype18, add49)
add53: R.Tensor((1, 1, 512), dtype="float16") = R.add(add52, add47)
layer_norm19: R.Tensor((1, 1, 512), dtype="float16") = R.nn.layer_norm(add53, gpt_neox_layers_3_input_layernorm_weight2, gpt_neox_layers_3_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv48 = R.call_tir(cls.dequantize1, (gpt_neox_layers_3_attention_query_key_value_q_weight2, gpt_neox_layers_3_attention_query_key_value_q_scale2), out_sinfo=R.Tensor((1536, 512), dtype="float16"))
permute_dims37: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv48, axes=None)
matmul37: R.Tensor((1, 1, 1536), dtype="float16") = R.matmul(layer_norm19, permute_dims37, out_dtype="void")
add54: R.Tensor((1, 1, 1536), dtype="float16") = R.add(matmul37, gpt_neox_layers_3_attention_query_key_value_bias2)
reshape36: R.Tensor((1, 1, 24, 64), dtype="float16") = R.reshape(add54, R.shape([1, 1, 24, 64]))
reshape37: R.Tensor((1, 24, 64), dtype="float16") = R.reshape(reshape36, R.shape([1, 24, 64]))
lv49 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape37), out_sinfo=R.Tensor((1, 8, 64), dtype="float16"))
reshape38: R.Tensor((1, 1, 8, 64), dtype="float16") = R.reshape(lv49, R.shape([1, 1, 8, 64]))
reshape39: R.Tensor((1, 1, 512), dtype="float16") = R.reshape(reshape38, R.shape([1, 1, 512]))
lv50 = R.call_tir(cls.dequantize2, (gpt_neox_layers_3_attention_dense_q_weight2, gpt_neox_layers_3_attention_dense_q_scale2), out_sinfo=R.Tensor((512, 512), dtype="float16"))
permute_dims38: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv50, axes=None)
matmul38: R.Tensor((1, 1, 512), dtype="float16") = R.matmul(reshape39, permute_dims38, out_dtype="void")
add55: R.Tensor((1, 1, 512), dtype="float16") = R.add(matmul38, gpt_neox_layers_3_attention_dense_bias2)
layer_norm20: R.Tensor((1, 1, 512), dtype="float16") = R.nn.layer_norm(add53, gpt_neox_layers_3_post_attention_layernorm_weight2, gpt_neox_layers_3_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv51 = R.call_tir(cls.dequantize3, (gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale2), out_sinfo=R.Tensor((2048, 512), dtype="float16"))
permute_dims39: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv51, axes=None)
matmul39: R.Tensor((1, 1, 2048), dtype="float32") = R.matmul(layer_norm20, permute_dims39, out_dtype="float32")
add56: R.Tensor((1, 1, 2048), dtype="float32") = R.add(matmul39, gpt_neox_layers_3_mlp_dense_h_to_4h_bias2)
gelu9: R.Tensor((1, 1, 2048), dtype="float32") = R.nn.gelu(add56)
astype19: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(gelu9, dtype="float16")
lv52 = R.call_tir(cls.dequantize4, (gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale2), out_sinfo=R.Tensor((512, 2048), dtype="float16"))
permute_dims40: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv52, axes=None)
matmul40: R.Tensor((1, 1, 512), dtype="float32") = R.matmul(astype19, permute_dims40, out_dtype="float32")
add57: R.Tensor((1, 1, 512), dtype="float32") = R.add(matmul40, gpt_neox_layers_3_mlp_dense_4h_to_h_bias2)
astype20: R.Tensor((1, 1, 512), dtype="float16") = R.astype(add57, dtype="float16")
add58: R.Tensor((1, 1, 512), dtype="float16") = R.add(astype20, add55)
add59: R.Tensor((1, 1, 512), dtype="float16") = R.add(add58, add53)
layer_norm21: R.Tensor((1, 1, 512), dtype="float16") = R.nn.layer_norm(add59, gpt_neox_layers_4_input_layernorm_weight2, gpt_neox_layers_4_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv53 = R.call_tir(cls.dequantize1, (gpt_neox_layers_4_attention_query_key_value_q_weight2, gpt_neox_layers_4_attention_query_key_value_q_scale2), out_sinfo=R.Tensor((1536, 512), dtype="float16"))
permute_dims41: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv53, axes=None)
matmul41: R.Tensor((1, 1, 1536), dtype="float16") = R.matmul(layer_norm21, permute_dims41, out_dtype="void")
add60: R.Tensor((1, 1, 1536), dtype="float16") = R.add(matmul41, gpt_neox_layers_4_attention_query_key_value_bias2)
reshape40: R.Tensor((1, 1, 24, 64), dtype="float16") = R.reshape(add60, R.shape([1, 1, 24, 64]))
reshape41: R.Tensor((1, 24, 64), dtype="float16") = R.reshape(reshape40, R.shape([1, 24, 64]))
lv54 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape41), out_sinfo=R.Tensor((1, 8, 64), dtype="float16"))
reshape42: R.Tensor((1, 1, 8, 64), dtype="float16") = R.reshape(lv54, R.shape([1, 1, 8, 64]))
reshape43: R.Tensor((1, 1, 512), dtype="float16") = R.reshape(reshape42, R.shape([1, 1, 512]))
lv55 = R.call_tir(cls.dequantize2, (gpt_neox_layers_4_attention_dense_q_weight2, gpt_neox_layers_4_attention_dense_q_scale2), out_sinfo=R.Tensor((512, 512), dtype="float16"))
permute_dims42: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv55, axes=None)
matmul42: R.Tensor((1, 1, 512), dtype="float16") = R.matmul(reshape43, permute_dims42, out_dtype="void")
add61: R.Tensor((1, 1, 512), dtype="float16") = R.add(matmul42, gpt_neox_layers_4_attention_dense_bias2)
layer_norm22: R.Tensor((1, 1, 512), dtype="float16") = R.nn.layer_norm(add59, gpt_neox_layers_4_post_attention_layernorm_weight2, gpt_neox_layers_4_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv56 = R.call_tir(cls.dequantize3, (gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale2), out_sinfo=R.Tensor((2048, 512), dtype="float16"))
permute_dims43: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv56, axes=None)
matmul43: R.Tensor((1, 1, 2048), dtype="float32") = R.matmul(layer_norm22, permute_dims43, out_dtype="float32")
add62: R.Tensor((1, 1, 2048), dtype="float32") = R.add(matmul43, gpt_neox_layers_4_mlp_dense_h_to_4h_bias2)
gelu10: R.Tensor((1, 1, 2048), dtype="float32") = R.nn.gelu(add62)
astype21: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(gelu10, dtype="float16")
lv57 = R.call_tir(cls.dequantize4, (gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale2), out_sinfo=R.Tensor((512, 2048), dtype="float16"))
permute_dims44: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv57, axes=None)
matmul44: R.Tensor((1, 1, 512), dtype="float32") = R.matmul(astype21, permute_dims44, out_dtype="float32")
add63: R.Tensor((1, 1, 512), dtype="float32") = R.add(matmul44, gpt_neox_layers_4_mlp_dense_4h_to_h_bias2)
astype22: R.Tensor((1, 1, 512), dtype="float16") = R.astype(add63, dtype="float16")
add64: R.Tensor((1, 1, 512), dtype="float16") = R.add(astype22, add61)
add65: R.Tensor((1, 1, 512), dtype="float16") = R.add(add64, add59)
layer_norm23: R.Tensor((1, 1, 512), dtype="float16") = R.nn.layer_norm(add65, gpt_neox_layers_5_input_layernorm_weight2, gpt_neox_layers_5_input_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv58 = R.call_tir(cls.dequantize1, (gpt_neox_layers_5_attention_query_key_value_q_weight2, gpt_neox_layers_5_attention_query_key_value_q_scale2), out_sinfo=R.Tensor((1536, 512), dtype="float16"))
permute_dims45: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv58, axes=None)
matmul45: R.Tensor((1, 1, 1536), dtype="float16") = R.matmul(layer_norm23, permute_dims45, out_dtype="void")
add66: R.Tensor((1, 1, 1536), dtype="float16") = R.add(matmul45, gpt_neox_layers_5_attention_query_key_value_bias2)
reshape44: R.Tensor((1, 1, 24, 64), dtype="float16") = R.reshape(add66, R.shape([1, 1, 24, 64]))
reshape45: R.Tensor((1, 24, 64), dtype="float16") = R.reshape(reshape44, R.shape([1, 24, 64]))
lv59 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape45), out_sinfo=R.Tensor((1, 8, 64), dtype="float16"))
reshape46: R.Tensor((1, 1, 8, 64), dtype="float16") = R.reshape(lv59, R.shape([1, 1, 8, 64]))
reshape47: R.Tensor((1, 1, 512), dtype="float16") = R.reshape(reshape46, R.shape([1, 1, 512]))
lv60 = R.call_tir(cls.dequantize2, (gpt_neox_layers_5_attention_dense_q_weight2, gpt_neox_layers_5_attention_dense_q_scale2), out_sinfo=R.Tensor((512, 512), dtype="float16"))
permute_dims46: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv60, axes=None)
matmul46: R.Tensor((1, 1, 512), dtype="float16") = R.matmul(reshape47, permute_dims46, out_dtype="void")
add67: R.Tensor((1, 1, 512), dtype="float16") = R.add(matmul46, gpt_neox_layers_5_attention_dense_bias2)
layer_norm24: R.Tensor((1, 1, 512), dtype="float16") = R.nn.layer_norm(add65, gpt_neox_layers_5_post_attention_layernorm_weight2, gpt_neox_layers_5_post_attention_layernorm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv61 = R.call_tir(cls.dequantize3, (gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight2, gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale2), out_sinfo=R.Tensor((2048, 512), dtype="float16"))
permute_dims47: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv61, axes=None)
matmul47: R.Tensor((1, 1, 2048), dtype="float32") = R.matmul(layer_norm24, permute_dims47, out_dtype="float32")
add68: R.Tensor((1, 1, 2048), dtype="float32") = R.add(matmul47, gpt_neox_layers_5_mlp_dense_h_to_4h_bias2)
gelu11: R.Tensor((1, 1, 2048), dtype="float32") = R.nn.gelu(add68)
astype23: R.Tensor((1, 1, 2048), dtype="float16") = R.astype(gelu11, dtype="float16")
lv62 = R.call_tir(cls.dequantize4, (gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight2, gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale2), out_sinfo=R.Tensor((512, 2048), dtype="float16"))
permute_dims48: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv62, axes=None)
matmul48: R.Tensor((1, 1, 512), dtype="float32") = R.matmul(astype23, permute_dims48, out_dtype="float32")
add69: R.Tensor((1, 1, 512), dtype="float32") = R.add(matmul48, gpt_neox_layers_5_mlp_dense_4h_to_h_bias2)
astype24: R.Tensor((1, 1, 512), dtype="float16") = R.astype(add69, dtype="float16")
add70: R.Tensor((1, 1, 512), dtype="float16") = R.add(astype24, add67)
add71: R.Tensor((1, 1, 512), dtype="float16") = R.add(add70, add65)
layer_norm25: R.Tensor((1, 1, 512), dtype="float16") = R.nn.layer_norm(add71, gpt_neox_final_layer_norm_weight2, gpt_neox_final_layer_norm_bias2, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv63 = R.call_tir(cls.dequantize, (embed_out_q_weight2, embed_out_q_scale2), out_sinfo=R.Tensor((vocab_size, 512), dtype="float16"))
permute_dims49: R.Tensor((512, vocab_size), dtype="float16") = R.permute_dims(lv63, axes=None)
matmul49: R.Tensor((1, 1, vocab_size), dtype="float16") = R.matmul(layer_norm25, permute_dims49, out_dtype="void")
astype25: R.Tensor((1, 1, vocab_size), dtype="float32") = R.astype(matmul49, dtype="float32")
gv2: R.Tuple(R.Tensor((1, 1, vocab_size), dtype="float32"), R.Object) = astype25, paged_kv_cache
R.output(gv2)
return gv2
@R.function
def embed(input_ids: R.Tensor(("seq_len",), dtype="int32"), packed_params: R.Tuple(R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"))) -> R.Tensor(("seq_len", 512), dtype="float16"):
seq_len = T.int64()
vocab_size = T.int64()
R.func_attr({"num_input": 1, "relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}})
cls = Module
with R.dataflow():
gpt_neox_embed_in_q_weight: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[0]
gpt_neox_embed_in_q_scale: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[1]
gpt_neox_layers_0_input_layernorm_weight: R.Tensor((512,), dtype="float16") = packed_params[2]
gpt_neox_layers_0_input_layernorm_bias: R.Tensor((512,), dtype="float16") = packed_params[3]
gpt_neox_layers_0_post_attention_layernorm_weight: R.Tensor((512,), dtype="float16") = packed_params[4]
gpt_neox_layers_0_post_attention_layernorm_bias: R.Tensor((512,), dtype="float16") = packed_params[5]
gpt_neox_layers_0_attention_query_key_value_q_weight: R.Tensor((1536, 64), dtype="uint32") = packed_params[6]
gpt_neox_layers_0_attention_query_key_value_q_scale: R.Tensor((1536, 16), dtype="float16") = packed_params[7]
gpt_neox_layers_0_attention_query_key_value_bias: R.Tensor((1536,), dtype="float16") = packed_params[8]
gpt_neox_layers_0_attention_dense_q_weight: R.Tensor((512, 64), dtype="uint32") = packed_params[9]
gpt_neox_layers_0_attention_dense_q_scale: R.Tensor((512, 16), dtype="float16") = packed_params[10]
gpt_neox_layers_0_attention_dense_bias: R.Tensor((512,), dtype="float16") = packed_params[11]
gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight: R.Tensor((2048, 64), dtype="uint32") = packed_params[12]
gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale: R.Tensor((2048, 16), dtype="float16") = packed_params[13]
gpt_neox_layers_0_mlp_dense_h_to_4h_bias: R.Tensor((2048,), dtype="float32") = packed_params[14]
gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight: R.Tensor((512, 256), dtype="uint32") = packed_params[15]
gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale: R.Tensor((512, 64), dtype="float16") = packed_params[16]
gpt_neox_layers_0_mlp_dense_4h_to_h_bias: R.Tensor((512,), dtype="float32") = packed_params[17]
gpt_neox_layers_1_input_layernorm_weight: R.Tensor((512,), dtype="float16") = packed_params[18]
gpt_neox_layers_1_input_layernorm_bias: R.Tensor((512,), dtype="float16") = packed_params[19]
gpt_neox_layers_1_post_attention_layernorm_weight: R.Tensor((512,), dtype="float16") = packed_params[20]
gpt_neox_layers_1_post_attention_layernorm_bias: R.Tensor((512,), dtype="float16") = packed_params[21]
gpt_neox_layers_1_attention_query_key_value_q_weight: R.Tensor((1536, 64), dtype="uint32") = packed_params[22]
gpt_neox_layers_1_attention_query_key_value_q_scale: R.Tensor((1536, 16), dtype="float16") = packed_params[23]
gpt_neox_layers_1_attention_query_key_value_bias: R.Tensor((1536,), dtype="float16") = packed_params[24]
gpt_neox_layers_1_attention_dense_q_weight: R.Tensor((512, 64), dtype="uint32") = packed_params[25]
gpt_neox_layers_1_attention_dense_q_scale: R.Tensor((512, 16), dtype="float16") = packed_params[26]
gpt_neox_layers_1_attention_dense_bias: R.Tensor((512,), dtype="float16") = packed_params[27]
gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight: R.Tensor((2048, 64), dtype="uint32") = packed_params[28]
gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale: R.Tensor((2048, 16), dtype="float16") = packed_params[29]
gpt_neox_layers_1_mlp_dense_h_to_4h_bias: R.Tensor((2048,), dtype="float32") = packed_params[30]
gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight: R.Tensor((512, 256), dtype="uint32") = packed_params[31]
gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale: R.Tensor((512, 64), dtype="float16") = packed_params[32]
gpt_neox_layers_1_mlp_dense_4h_to_h_bias: R.Tensor((512,), dtype="float32") = packed_params[33]
gpt_neox_layers_2_input_layernorm_weight: R.Tensor((512,), dtype="float16") = packed_params[34]
gpt_neox_layers_2_input_layernorm_bias: R.Tensor((512,), dtype="float16") = packed_params[35]
gpt_neox_layers_2_post_attention_layernorm_weight: R.Tensor((512,), dtype="float16") = packed_params[36]
gpt_neox_layers_2_post_attention_layernorm_bias: R.Tensor((512,), dtype="float16") = packed_params[37]
gpt_neox_layers_2_attention_query_key_value_q_weight: R.Tensor((1536, 64), dtype="uint32") = packed_params[38]
gpt_neox_layers_2_attention_query_key_value_q_scale: R.Tensor((1536, 16), dtype="float16") = packed_params[39]
gpt_neox_layers_2_attention_query_key_value_bias: R.Tensor((1536,), dtype="float16") = packed_params[40]
gpt_neox_layers_2_attention_dense_q_weight: R.Tensor((512, 64), dtype="uint32") = packed_params[41]
gpt_neox_layers_2_attention_dense_q_scale: R.Tensor((512, 16), dtype="float16") = packed_params[42]
gpt_neox_layers_2_attention_dense_bias: R.Tensor((512,), dtype="float16") = packed_params[43]
gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight: R.Tensor((2048, 64), dtype="uint32") = packed_params[44]
gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale: R.Tensor((2048, 16), dtype="float16") = packed_params[45]
gpt_neox_layers_2_mlp_dense_h_to_4h_bias: R.Tensor((2048,), dtype="float32") = packed_params[46]
gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight: R.Tensor((512, 256), dtype="uint32") = packed_params[47]
gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale: R.Tensor((512, 64), dtype="float16") = packed_params[48]
gpt_neox_layers_2_mlp_dense_4h_to_h_bias: R.Tensor((512,), dtype="float32") = packed_params[49]
gpt_neox_layers_3_input_layernorm_weight: R.Tensor((512,), dtype="float16") = packed_params[50]
gpt_neox_layers_3_input_layernorm_bias: R.Tensor((512,), dtype="float16") = packed_params[51]
gpt_neox_layers_3_post_attention_layernorm_weight: R.Tensor((512,), dtype="float16") = packed_params[52]
gpt_neox_layers_3_post_attention_layernorm_bias: R.Tensor((512,), dtype="float16") = packed_params[53]
gpt_neox_layers_3_attention_query_key_value_q_weight: R.Tensor((1536, 64), dtype="uint32") = packed_params[54]
gpt_neox_layers_3_attention_query_key_value_q_scale: R.Tensor((1536, 16), dtype="float16") = packed_params[55]
gpt_neox_layers_3_attention_query_key_value_bias: R.Tensor((1536,), dtype="float16") = packed_params[56]
gpt_neox_layers_3_attention_dense_q_weight: R.Tensor((512, 64), dtype="uint32") = packed_params[57]
gpt_neox_layers_3_attention_dense_q_scale: R.Tensor((512, 16), dtype="float16") = packed_params[58]
gpt_neox_layers_3_attention_dense_bias: R.Tensor((512,), dtype="float16") = packed_params[59]
gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight: R.Tensor((2048, 64), dtype="uint32") = packed_params[60]
gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale: R.Tensor((2048, 16), dtype="float16") = packed_params[61]
gpt_neox_layers_3_mlp_dense_h_to_4h_bias: R.Tensor((2048,), dtype="float32") = packed_params[62]
gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight: R.Tensor((512, 256), dtype="uint32") = packed_params[63]
gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale: R.Tensor((512, 64), dtype="float16") = packed_params[64]
gpt_neox_layers_3_mlp_dense_4h_to_h_bias: R.Tensor((512,), dtype="float32") = packed_params[65]
gpt_neox_layers_4_input_layernorm_weight: R.Tensor((512,), dtype="float16") = packed_params[66]
gpt_neox_layers_4_input_layernorm_bias: R.Tensor((512,), dtype="float16") = packed_params[67]
gpt_neox_layers_4_post_attention_layernorm_weight: R.Tensor((512,), dtype="float16") = packed_params[68]
gpt_neox_layers_4_post_attention_layernorm_bias: R.Tensor((512,), dtype="float16") = packed_params[69]
gpt_neox_layers_4_attention_query_key_value_q_weight: R.Tensor((1536, 64), dtype="uint32") = packed_params[70]
gpt_neox_layers_4_attention_query_key_value_q_scale: R.Tensor((1536, 16), dtype="float16") = packed_params[71]
gpt_neox_layers_4_attention_query_key_value_bias: R.Tensor((1536,), dtype="float16") = packed_params[72]
gpt_neox_layers_4_attention_dense_q_weight: R.Tensor((512, 64), dtype="uint32") = packed_params[73]
gpt_neox_layers_4_attention_dense_q_scale: R.Tensor((512, 16), dtype="float16") = packed_params[74]
gpt_neox_layers_4_attention_dense_bias: R.Tensor((512,), dtype="float16") = packed_params[75]
gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight: R.Tensor((2048, 64), dtype="uint32") = packed_params[76]
gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale: R.Tensor((2048, 16), dtype="float16") = packed_params[77]
gpt_neox_layers_4_mlp_dense_h_to_4h_bias: R.Tensor((2048,), dtype="float32") = packed_params[78]
gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight: R.Tensor((512, 256), dtype="uint32") = packed_params[79]
gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale: R.Tensor((512, 64), dtype="float16") = packed_params[80]
gpt_neox_layers_4_mlp_dense_4h_to_h_bias: R.Tensor((512,), dtype="float32") = packed_params[81]
gpt_neox_layers_5_input_layernorm_weight: R.Tensor((512,), dtype="float16") = packed_params[82]
gpt_neox_layers_5_input_layernorm_bias: R.Tensor((512,), dtype="float16") = packed_params[83]
gpt_neox_layers_5_post_attention_layernorm_weight: R.Tensor((512,), dtype="float16") = packed_params[84]
gpt_neox_layers_5_post_attention_layernorm_bias: R.Tensor((512,), dtype="float16") = packed_params[85]
gpt_neox_layers_5_attention_query_key_value_q_weight: R.Tensor((1536, 64), dtype="uint32") = packed_params[86]
gpt_neox_layers_5_attention_query_key_value_q_scale: R.Tensor((1536, 16), dtype="float16") = packed_params[87]
gpt_neox_layers_5_attention_query_key_value_bias: R.Tensor((1536,), dtype="float16") = packed_params[88]
gpt_neox_layers_5_attention_dense_q_weight: R.Tensor((512, 64), dtype="uint32") = packed_params[89]
gpt_neox_layers_5_attention_dense_q_scale: R.Tensor((512, 16), dtype="float16") = packed_params[90]
gpt_neox_layers_5_attention_dense_bias: R.Tensor((512,), dtype="float16") = packed_params[91]
gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight: R.Tensor((2048, 64), dtype="uint32") = packed_params[92]
gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale: R.Tensor((2048, 16), dtype="float16") = packed_params[93]
gpt_neox_layers_5_mlp_dense_h_to_4h_bias: R.Tensor((2048,), dtype="float32") = packed_params[94]
gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight: R.Tensor((512, 256), dtype="uint32") = packed_params[95]
gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale: R.Tensor((512, 64), dtype="float16") = packed_params[96]
gpt_neox_layers_5_mlp_dense_4h_to_h_bias: R.Tensor((512,), dtype="float32") = packed_params[97]
gpt_neox_final_layer_norm_weight: R.Tensor((512,), dtype="float16") = packed_params[98]
gpt_neox_final_layer_norm_bias: R.Tensor((512,), dtype="float16") = packed_params[99]
embed_out_q_weight: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[100]
embed_out_q_scale: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[101]
lv = R.call_tir(cls.dequantize, (gpt_neox_embed_in_q_weight, gpt_neox_embed_in_q_scale), out_sinfo=R.Tensor((vocab_size, 512), dtype="float16"))
take: R.Tensor((seq_len, 512), dtype="float16") = R.take(lv, input_ids, axis=0)
gv: R.Tensor((seq_len, 512), dtype="float16") = take
R.output(gv)
return gv
@R.function
def prefill(input_embed: R.Tensor((1, "seq_len", 512), dtype="float16"), paged_kv_cache: R.Object, packed_params: R.Tuple(R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((1536, 64), dtype="uint32"), R.Tensor((1536, 16), dtype="float16"), R.Tensor((1536,), dtype="float16"), R.Tensor((512, 64), dtype="uint32"), R.Tensor((512, 16), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor((2048, 64), dtype="uint32"), R.Tensor((2048, 16), dtype="float16"), R.Tensor((2048,), dtype="float32"), R.Tensor((512, 256), dtype="uint32"), R.Tensor((512, 64), dtype="float16"), R.Tensor((512,), dtype="float32"), R.Tensor((512,), dtype="float16"), R.Tensor((512,), dtype="float16"), R.Tensor(("vocab_size", 64), dtype="uint32"), R.Tensor(("vocab_size", 16), dtype="float16"))) -> R.Tuple(R.Tensor((1, 1, "vocab_size"), dtype="float32"), R.Object):
vocab_size = T.int64()
seq_len = T.int64()
R.func_attr({"num_input": 2, "pipeline_parallel_stages": 1, "relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}})
cls = Module
with R.dataflow():
gpt_neox_embed_in_q_weight1: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[0]
gpt_neox_embed_in_q_scale1: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[1]
gpt_neox_layers_0_input_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[2]
gpt_neox_layers_0_input_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[3]
gpt_neox_layers_0_post_attention_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[4]
gpt_neox_layers_0_post_attention_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[5]
gpt_neox_layers_0_attention_query_key_value_q_weight1: R.Tensor((1536, 64), dtype="uint32") = packed_params[6]
gpt_neox_layers_0_attention_query_key_value_q_scale1: R.Tensor((1536, 16), dtype="float16") = packed_params[7]
gpt_neox_layers_0_attention_query_key_value_bias1: R.Tensor((1536,), dtype="float16") = packed_params[8]
gpt_neox_layers_0_attention_dense_q_weight1: R.Tensor((512, 64), dtype="uint32") = packed_params[9]
gpt_neox_layers_0_attention_dense_q_scale1: R.Tensor((512, 16), dtype="float16") = packed_params[10]
gpt_neox_layers_0_attention_dense_bias1: R.Tensor((512,), dtype="float16") = packed_params[11]
gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight1: R.Tensor((2048, 64), dtype="uint32") = packed_params[12]
gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale1: R.Tensor((2048, 16), dtype="float16") = packed_params[13]
gpt_neox_layers_0_mlp_dense_h_to_4h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[14]
gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight1: R.Tensor((512, 256), dtype="uint32") = packed_params[15]
gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale1: R.Tensor((512, 64), dtype="float16") = packed_params[16]
gpt_neox_layers_0_mlp_dense_4h_to_h_bias1: R.Tensor((512,), dtype="float32") = packed_params[17]
gpt_neox_layers_1_input_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[18]
gpt_neox_layers_1_input_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[19]
gpt_neox_layers_1_post_attention_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[20]
gpt_neox_layers_1_post_attention_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[21]
gpt_neox_layers_1_attention_query_key_value_q_weight1: R.Tensor((1536, 64), dtype="uint32") = packed_params[22]
gpt_neox_layers_1_attention_query_key_value_q_scale1: R.Tensor((1536, 16), dtype="float16") = packed_params[23]
gpt_neox_layers_1_attention_query_key_value_bias1: R.Tensor((1536,), dtype="float16") = packed_params[24]
gpt_neox_layers_1_attention_dense_q_weight1: R.Tensor((512, 64), dtype="uint32") = packed_params[25]
gpt_neox_layers_1_attention_dense_q_scale1: R.Tensor((512, 16), dtype="float16") = packed_params[26]
gpt_neox_layers_1_attention_dense_bias1: R.Tensor((512,), dtype="float16") = packed_params[27]
gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight1: R.Tensor((2048, 64), dtype="uint32") = packed_params[28]
gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale1: R.Tensor((2048, 16), dtype="float16") = packed_params[29]
gpt_neox_layers_1_mlp_dense_h_to_4h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[30]
gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight1: R.Tensor((512, 256), dtype="uint32") = packed_params[31]
gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale1: R.Tensor((512, 64), dtype="float16") = packed_params[32]
gpt_neox_layers_1_mlp_dense_4h_to_h_bias1: R.Tensor((512,), dtype="float32") = packed_params[33]
gpt_neox_layers_2_input_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[34]
gpt_neox_layers_2_input_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[35]
gpt_neox_layers_2_post_attention_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[36]
gpt_neox_layers_2_post_attention_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[37]
gpt_neox_layers_2_attention_query_key_value_q_weight1: R.Tensor((1536, 64), dtype="uint32") = packed_params[38]
gpt_neox_layers_2_attention_query_key_value_q_scale1: R.Tensor((1536, 16), dtype="float16") = packed_params[39]
gpt_neox_layers_2_attention_query_key_value_bias1: R.Tensor((1536,), dtype="float16") = packed_params[40]
gpt_neox_layers_2_attention_dense_q_weight1: R.Tensor((512, 64), dtype="uint32") = packed_params[41]
gpt_neox_layers_2_attention_dense_q_scale1: R.Tensor((512, 16), dtype="float16") = packed_params[42]
gpt_neox_layers_2_attention_dense_bias1: R.Tensor((512,), dtype="float16") = packed_params[43]
gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight1: R.Tensor((2048, 64), dtype="uint32") = packed_params[44]
gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale1: R.Tensor((2048, 16), dtype="float16") = packed_params[45]
gpt_neox_layers_2_mlp_dense_h_to_4h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[46]
gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight1: R.Tensor((512, 256), dtype="uint32") = packed_params[47]
gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale1: R.Tensor((512, 64), dtype="float16") = packed_params[48]
gpt_neox_layers_2_mlp_dense_4h_to_h_bias1: R.Tensor((512,), dtype="float32") = packed_params[49]
gpt_neox_layers_3_input_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[50]
gpt_neox_layers_3_input_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[51]
gpt_neox_layers_3_post_attention_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[52]
gpt_neox_layers_3_post_attention_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[53]
gpt_neox_layers_3_attention_query_key_value_q_weight1: R.Tensor((1536, 64), dtype="uint32") = packed_params[54]
gpt_neox_layers_3_attention_query_key_value_q_scale1: R.Tensor((1536, 16), dtype="float16") = packed_params[55]
gpt_neox_layers_3_attention_query_key_value_bias1: R.Tensor((1536,), dtype="float16") = packed_params[56]
gpt_neox_layers_3_attention_dense_q_weight1: R.Tensor((512, 64), dtype="uint32") = packed_params[57]
gpt_neox_layers_3_attention_dense_q_scale1: R.Tensor((512, 16), dtype="float16") = packed_params[58]
gpt_neox_layers_3_attention_dense_bias1: R.Tensor((512,), dtype="float16") = packed_params[59]
gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight1: R.Tensor((2048, 64), dtype="uint32") = packed_params[60]
gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale1: R.Tensor((2048, 16), dtype="float16") = packed_params[61]
gpt_neox_layers_3_mlp_dense_h_to_4h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[62]
gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight1: R.Tensor((512, 256), dtype="uint32") = packed_params[63]
gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale1: R.Tensor((512, 64), dtype="float16") = packed_params[64]
gpt_neox_layers_3_mlp_dense_4h_to_h_bias1: R.Tensor((512,), dtype="float32") = packed_params[65]
gpt_neox_layers_4_input_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[66]
gpt_neox_layers_4_input_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[67]
gpt_neox_layers_4_post_attention_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[68]
gpt_neox_layers_4_post_attention_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[69]
gpt_neox_layers_4_attention_query_key_value_q_weight1: R.Tensor((1536, 64), dtype="uint32") = packed_params[70]
gpt_neox_layers_4_attention_query_key_value_q_scale1: R.Tensor((1536, 16), dtype="float16") = packed_params[71]
gpt_neox_layers_4_attention_query_key_value_bias1: R.Tensor((1536,), dtype="float16") = packed_params[72]
gpt_neox_layers_4_attention_dense_q_weight1: R.Tensor((512, 64), dtype="uint32") = packed_params[73]
gpt_neox_layers_4_attention_dense_q_scale1: R.Tensor((512, 16), dtype="float16") = packed_params[74]
gpt_neox_layers_4_attention_dense_bias1: R.Tensor((512,), dtype="float16") = packed_params[75]
gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight1: R.Tensor((2048, 64), dtype="uint32") = packed_params[76]
gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale1: R.Tensor((2048, 16), dtype="float16") = packed_params[77]
gpt_neox_layers_4_mlp_dense_h_to_4h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[78]
gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight1: R.Tensor((512, 256), dtype="uint32") = packed_params[79]
gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale1: R.Tensor((512, 64), dtype="float16") = packed_params[80]
gpt_neox_layers_4_mlp_dense_4h_to_h_bias1: R.Tensor((512,), dtype="float32") = packed_params[81]
gpt_neox_layers_5_input_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[82]
gpt_neox_layers_5_input_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[83]
gpt_neox_layers_5_post_attention_layernorm_weight1: R.Tensor((512,), dtype="float16") = packed_params[84]
gpt_neox_layers_5_post_attention_layernorm_bias1: R.Tensor((512,), dtype="float16") = packed_params[85]
gpt_neox_layers_5_attention_query_key_value_q_weight1: R.Tensor((1536, 64), dtype="uint32") = packed_params[86]
gpt_neox_layers_5_attention_query_key_value_q_scale1: R.Tensor((1536, 16), dtype="float16") = packed_params[87]
gpt_neox_layers_5_attention_query_key_value_bias1: R.Tensor((1536,), dtype="float16") = packed_params[88]
gpt_neox_layers_5_attention_dense_q_weight1: R.Tensor((512, 64), dtype="uint32") = packed_params[89]
gpt_neox_layers_5_attention_dense_q_scale1: R.Tensor((512, 16), dtype="float16") = packed_params[90]
gpt_neox_layers_5_attention_dense_bias1: R.Tensor((512,), dtype="float16") = packed_params[91]
gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight1: R.Tensor((2048, 64), dtype="uint32") = packed_params[92]
gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale1: R.Tensor((2048, 16), dtype="float16") = packed_params[93]
gpt_neox_layers_5_mlp_dense_h_to_4h_bias1: R.Tensor((2048,), dtype="float32") = packed_params[94]
gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight1: R.Tensor((512, 256), dtype="uint32") = packed_params[95]
gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale1: R.Tensor((512, 64), dtype="float16") = packed_params[96]
gpt_neox_layers_5_mlp_dense_4h_to_h_bias1: R.Tensor((512,), dtype="float32") = packed_params[97]
gpt_neox_final_layer_norm_weight1: R.Tensor((512,), dtype="float16") = packed_params[98]
gpt_neox_final_layer_norm_bias1: R.Tensor((512,), dtype="float16") = packed_params[99]
embed_out_q_weight1: R.Tensor((vocab_size, 64), dtype="uint32") = packed_params[100]
embed_out_q_scale1: R.Tensor((vocab_size, 16), dtype="float16") = packed_params[101]
layer_norm: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(input_embed, gpt_neox_layers_0_input_layernorm_weight1, gpt_neox_layers_0_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv1 = R.call_tir(cls.dequantize1, (gpt_neox_layers_0_attention_query_key_value_q_weight1, gpt_neox_layers_0_attention_query_key_value_q_scale1), out_sinfo=R.Tensor((1536, 512), dtype="float16"))
permute_dims: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv1, axes=None)
matmul: R.Tensor((1, seq_len, 1536), dtype="float16") = R.matmul(layer_norm, permute_dims, out_dtype="void")
add: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(matmul, gpt_neox_layers_0_attention_query_key_value_bias1)
reshape: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add, R.shape([1, seq_len, 24, 64]))
reshape1: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape, R.shape([seq_len, 24, 64]))
lv2 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(0), R.prim_value(T.float32(1.0)), reshape1), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16"))
reshape2: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv2, R.shape([1, seq_len, 8, 64]))
reshape3: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape2, R.shape([1, seq_len, 512]))
lv3 = R.call_tir(cls.dequantize2, (gpt_neox_layers_0_attention_dense_q_weight1, gpt_neox_layers_0_attention_dense_q_scale1), out_sinfo=R.Tensor((512, 512), dtype="float16"))
permute_dims1: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv3, axes=None)
matmul1: R.Tensor((1, seq_len, 512), dtype="float16") = R.matmul(reshape3, permute_dims1, out_dtype="void")
add1: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(matmul1, gpt_neox_layers_0_attention_dense_bias1)
layer_norm1: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(input_embed, gpt_neox_layers_0_post_attention_layernorm_weight1, gpt_neox_layers_0_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv4 = R.call_tir(cls.dequantize3, (gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale1), out_sinfo=R.Tensor((2048, 512), dtype="float16"))
permute_dims2: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv4, axes=None)
matmul2: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(layer_norm1, permute_dims2, out_dtype="float32")
add2: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul2, gpt_neox_layers_0_mlp_dense_h_to_4h_bias1)
gelu: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add2)
astype: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu, dtype="float16")
lv5 = R.call_tir(cls.dequantize4, (gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale1), out_sinfo=R.Tensor((512, 2048), dtype="float16"))
permute_dims3: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv5, axes=None)
matmul3: R.Tensor((1, seq_len, 512), dtype="float32") = R.matmul(astype, permute_dims3, out_dtype="float32")
add3: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(matmul3, gpt_neox_layers_0_mlp_dense_4h_to_h_bias1)
astype1: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add3, dtype="float16")
add4: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype1, add1)
add5: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add4, input_embed)
layer_norm2: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add5, gpt_neox_layers_1_input_layernorm_weight1, gpt_neox_layers_1_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv6 = R.call_tir(cls.dequantize1, (gpt_neox_layers_1_attention_query_key_value_q_weight1, gpt_neox_layers_1_attention_query_key_value_q_scale1), out_sinfo=R.Tensor((1536, 512), dtype="float16"))
permute_dims4: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv6, axes=None)
matmul4: R.Tensor((1, seq_len, 1536), dtype="float16") = R.matmul(layer_norm2, permute_dims4, out_dtype="void")
add6: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(matmul4, gpt_neox_layers_1_attention_query_key_value_bias1)
reshape4: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add6, R.shape([1, seq_len, 24, 64]))
reshape5: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape4, R.shape([seq_len, 24, 64]))
lv7 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(1), R.prim_value(T.float32(1.0)), reshape5), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16"))
reshape6: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv7, R.shape([1, seq_len, 8, 64]))
reshape7: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape6, R.shape([1, seq_len, 512]))
lv8 = R.call_tir(cls.dequantize2, (gpt_neox_layers_1_attention_dense_q_weight1, gpt_neox_layers_1_attention_dense_q_scale1), out_sinfo=R.Tensor((512, 512), dtype="float16"))
permute_dims5: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv8, axes=None)
matmul5: R.Tensor((1, seq_len, 512), dtype="float16") = R.matmul(reshape7, permute_dims5, out_dtype="void")
add7: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(matmul5, gpt_neox_layers_1_attention_dense_bias1)
layer_norm3: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add5, gpt_neox_layers_1_post_attention_layernorm_weight1, gpt_neox_layers_1_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv9 = R.call_tir(cls.dequantize3, (gpt_neox_layers_1_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_1_mlp_dense_h_to_4h_q_scale1), out_sinfo=R.Tensor((2048, 512), dtype="float16"))
permute_dims6: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv9, axes=None)
matmul6: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(layer_norm3, permute_dims6, out_dtype="float32")
add8: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul6, gpt_neox_layers_1_mlp_dense_h_to_4h_bias1)
gelu1: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add8)
astype2: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu1, dtype="float16")
lv10 = R.call_tir(cls.dequantize4, (gpt_neox_layers_1_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_1_mlp_dense_4h_to_h_q_scale1), out_sinfo=R.Tensor((512, 2048), dtype="float16"))
permute_dims7: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv10, axes=None)
matmul7: R.Tensor((1, seq_len, 512), dtype="float32") = R.matmul(astype2, permute_dims7, out_dtype="float32")
add9: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(matmul7, gpt_neox_layers_1_mlp_dense_4h_to_h_bias1)
astype3: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add9, dtype="float16")
add10: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype3, add7)
add11: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add10, add5)
layer_norm4: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add11, gpt_neox_layers_2_input_layernorm_weight1, gpt_neox_layers_2_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv11 = R.call_tir(cls.dequantize1, (gpt_neox_layers_2_attention_query_key_value_q_weight1, gpt_neox_layers_2_attention_query_key_value_q_scale1), out_sinfo=R.Tensor((1536, 512), dtype="float16"))
permute_dims8: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv11, axes=None)
matmul8: R.Tensor((1, seq_len, 1536), dtype="float16") = R.matmul(layer_norm4, permute_dims8, out_dtype="void")
add12: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(matmul8, gpt_neox_layers_2_attention_query_key_value_bias1)
reshape8: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add12, R.shape([1, seq_len, 24, 64]))
reshape9: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape8, R.shape([seq_len, 24, 64]))
lv12 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(2), R.prim_value(T.float32(1.0)), reshape9), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16"))
reshape10: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv12, R.shape([1, seq_len, 8, 64]))
reshape11: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape10, R.shape([1, seq_len, 512]))
lv13 = R.call_tir(cls.dequantize2, (gpt_neox_layers_2_attention_dense_q_weight1, gpt_neox_layers_2_attention_dense_q_scale1), out_sinfo=R.Tensor((512, 512), dtype="float16"))
permute_dims9: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv13, axes=None)
matmul9: R.Tensor((1, seq_len, 512), dtype="float16") = R.matmul(reshape11, permute_dims9, out_dtype="void")
add13: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(matmul9, gpt_neox_layers_2_attention_dense_bias1)
layer_norm5: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add11, gpt_neox_layers_2_post_attention_layernorm_weight1, gpt_neox_layers_2_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv14 = R.call_tir(cls.dequantize3, (gpt_neox_layers_2_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_2_mlp_dense_h_to_4h_q_scale1), out_sinfo=R.Tensor((2048, 512), dtype="float16"))
permute_dims10: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv14, axes=None)
matmul10: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(layer_norm5, permute_dims10, out_dtype="float32")
add14: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul10, gpt_neox_layers_2_mlp_dense_h_to_4h_bias1)
gelu2: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add14)
astype4: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu2, dtype="float16")
lv15 = R.call_tir(cls.dequantize4, (gpt_neox_layers_2_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_2_mlp_dense_4h_to_h_q_scale1), out_sinfo=R.Tensor((512, 2048), dtype="float16"))
permute_dims11: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv15, axes=None)
matmul11: R.Tensor((1, seq_len, 512), dtype="float32") = R.matmul(astype4, permute_dims11, out_dtype="float32")
add15: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(matmul11, gpt_neox_layers_2_mlp_dense_4h_to_h_bias1)
astype5: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add15, dtype="float16")
add16: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype5, add13)
add17: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add16, add11)
layer_norm6: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add17, gpt_neox_layers_3_input_layernorm_weight1, gpt_neox_layers_3_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv16 = R.call_tir(cls.dequantize1, (gpt_neox_layers_3_attention_query_key_value_q_weight1, gpt_neox_layers_3_attention_query_key_value_q_scale1), out_sinfo=R.Tensor((1536, 512), dtype="float16"))
permute_dims12: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv16, axes=None)
matmul12: R.Tensor((1, seq_len, 1536), dtype="float16") = R.matmul(layer_norm6, permute_dims12, out_dtype="void")
add18: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(matmul12, gpt_neox_layers_3_attention_query_key_value_bias1)
reshape12: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add18, R.shape([1, seq_len, 24, 64]))
reshape13: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape12, R.shape([seq_len, 24, 64]))
lv17 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(3), R.prim_value(T.float32(1.0)), reshape13), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16"))
reshape14: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv17, R.shape([1, seq_len, 8, 64]))
reshape15: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape14, R.shape([1, seq_len, 512]))
lv18 = R.call_tir(cls.dequantize2, (gpt_neox_layers_3_attention_dense_q_weight1, gpt_neox_layers_3_attention_dense_q_scale1), out_sinfo=R.Tensor((512, 512), dtype="float16"))
permute_dims13: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv18, axes=None)
matmul13: R.Tensor((1, seq_len, 512), dtype="float16") = R.matmul(reshape15, permute_dims13, out_dtype="void")
add19: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(matmul13, gpt_neox_layers_3_attention_dense_bias1)
layer_norm7: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add17, gpt_neox_layers_3_post_attention_layernorm_weight1, gpt_neox_layers_3_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv19 = R.call_tir(cls.dequantize3, (gpt_neox_layers_3_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_3_mlp_dense_h_to_4h_q_scale1), out_sinfo=R.Tensor((2048, 512), dtype="float16"))
permute_dims14: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv19, axes=None)
matmul14: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(layer_norm7, permute_dims14, out_dtype="float32")
add20: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul14, gpt_neox_layers_3_mlp_dense_h_to_4h_bias1)
gelu3: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add20)
astype6: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu3, dtype="float16")
lv20 = R.call_tir(cls.dequantize4, (gpt_neox_layers_3_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_3_mlp_dense_4h_to_h_q_scale1), out_sinfo=R.Tensor((512, 2048), dtype="float16"))
permute_dims15: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv20, axes=None)
matmul15: R.Tensor((1, seq_len, 512), dtype="float32") = R.matmul(astype6, permute_dims15, out_dtype="float32")
add21: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(matmul15, gpt_neox_layers_3_mlp_dense_4h_to_h_bias1)
astype7: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add21, dtype="float16")
add22: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype7, add19)
add23: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add22, add17)
layer_norm8: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add23, gpt_neox_layers_4_input_layernorm_weight1, gpt_neox_layers_4_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv21 = R.call_tir(cls.dequantize1, (gpt_neox_layers_4_attention_query_key_value_q_weight1, gpt_neox_layers_4_attention_query_key_value_q_scale1), out_sinfo=R.Tensor((1536, 512), dtype="float16"))
permute_dims16: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv21, axes=None)
matmul16: R.Tensor((1, seq_len, 1536), dtype="float16") = R.matmul(layer_norm8, permute_dims16, out_dtype="void")
add24: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(matmul16, gpt_neox_layers_4_attention_query_key_value_bias1)
reshape16: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add24, R.shape([1, seq_len, 24, 64]))
reshape17: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape16, R.shape([seq_len, 24, 64]))
lv22 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(4), R.prim_value(T.float32(1.0)), reshape17), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16"))
reshape18: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv22, R.shape([1, seq_len, 8, 64]))
reshape19: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape18, R.shape([1, seq_len, 512]))
lv23 = R.call_tir(cls.dequantize2, (gpt_neox_layers_4_attention_dense_q_weight1, gpt_neox_layers_4_attention_dense_q_scale1), out_sinfo=R.Tensor((512, 512), dtype="float16"))
permute_dims17: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv23, axes=None)
matmul17: R.Tensor((1, seq_len, 512), dtype="float16") = R.matmul(reshape19, permute_dims17, out_dtype="void")
add25: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(matmul17, gpt_neox_layers_4_attention_dense_bias1)
layer_norm9: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add23, gpt_neox_layers_4_post_attention_layernorm_weight1, gpt_neox_layers_4_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv24 = R.call_tir(cls.dequantize3, (gpt_neox_layers_4_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_4_mlp_dense_h_to_4h_q_scale1), out_sinfo=R.Tensor((2048, 512), dtype="float16"))
permute_dims18: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv24, axes=None)
matmul18: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(layer_norm9, permute_dims18, out_dtype="float32")
add26: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul18, gpt_neox_layers_4_mlp_dense_h_to_4h_bias1)
gelu4: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add26)
astype8: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu4, dtype="float16")
lv25 = R.call_tir(cls.dequantize4, (gpt_neox_layers_4_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_4_mlp_dense_4h_to_h_q_scale1), out_sinfo=R.Tensor((512, 2048), dtype="float16"))
permute_dims19: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv25, axes=None)
matmul19: R.Tensor((1, seq_len, 512), dtype="float32") = R.matmul(astype8, permute_dims19, out_dtype="float32")
add27: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(matmul19, gpt_neox_layers_4_mlp_dense_4h_to_h_bias1)
astype9: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add27, dtype="float16")
add28: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype9, add25)
add29: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add28, add23)
layer_norm10: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add29, gpt_neox_layers_5_input_layernorm_weight1, gpt_neox_layers_5_input_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv26 = R.call_tir(cls.dequantize1, (gpt_neox_layers_5_attention_query_key_value_q_weight1, gpt_neox_layers_5_attention_query_key_value_q_scale1), out_sinfo=R.Tensor((1536, 512), dtype="float16"))
permute_dims20: R.Tensor((512, 1536), dtype="float16") = R.permute_dims(lv26, axes=None)
matmul20: R.Tensor((1, seq_len, 1536), dtype="float16") = R.matmul(layer_norm10, permute_dims20, out_dtype="void")
add30: R.Tensor((1, seq_len, 1536), dtype="float16") = R.add(matmul20, gpt_neox_layers_5_attention_query_key_value_bias1)
reshape20: R.Tensor((1, seq_len, 24, 64), dtype="float16") = R.reshape(add30, R.shape([1, seq_len, 24, 64]))
reshape21: R.Tensor((seq_len, 24, 64), dtype="float16") = R.reshape(reshape20, R.shape([seq_len, 24, 64]))
lv27 = R.call_dps_packed("vm.builtin.attention_kv_cache_attention_with_fused_qkv", (paged_kv_cache, R.prim_value(5), R.prim_value(T.float32(1.0)), reshape21), out_sinfo=R.Tensor((seq_len, 8, 64), dtype="float16"))
reshape22: R.Tensor((1, seq_len, 8, 64), dtype="float16") = R.reshape(lv27, R.shape([1, seq_len, 8, 64]))
reshape23: R.Tensor((1, seq_len, 512), dtype="float16") = R.reshape(reshape22, R.shape([1, seq_len, 512]))
lv28 = R.call_tir(cls.dequantize2, (gpt_neox_layers_5_attention_dense_q_weight1, gpt_neox_layers_5_attention_dense_q_scale1), out_sinfo=R.Tensor((512, 512), dtype="float16"))
permute_dims21: R.Tensor((512, 512), dtype="float16") = R.permute_dims(lv28, axes=None)
matmul21: R.Tensor((1, seq_len, 512), dtype="float16") = R.matmul(reshape23, permute_dims21, out_dtype="void")
add31: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(matmul21, gpt_neox_layers_5_attention_dense_bias1)
layer_norm11: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add29, gpt_neox_layers_5_post_attention_layernorm_weight1, gpt_neox_layers_5_post_attention_layernorm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv29 = R.call_tir(cls.dequantize3, (gpt_neox_layers_5_mlp_dense_h_to_4h_q_weight1, gpt_neox_layers_5_mlp_dense_h_to_4h_q_scale1), out_sinfo=R.Tensor((2048, 512), dtype="float16"))
permute_dims22: R.Tensor((512, 2048), dtype="float16") = R.permute_dims(lv29, axes=None)
matmul22: R.Tensor((1, seq_len, 2048), dtype="float32") = R.matmul(layer_norm11, permute_dims22, out_dtype="float32")
add32: R.Tensor((1, seq_len, 2048), dtype="float32") = R.add(matmul22, gpt_neox_layers_5_mlp_dense_h_to_4h_bias1)
gelu5: R.Tensor((1, seq_len, 2048), dtype="float32") = R.nn.gelu(add32)
astype10: R.Tensor((1, seq_len, 2048), dtype="float16") = R.astype(gelu5, dtype="float16")
lv30 = R.call_tir(cls.dequantize4, (gpt_neox_layers_5_mlp_dense_4h_to_h_q_weight1, gpt_neox_layers_5_mlp_dense_4h_to_h_q_scale1), out_sinfo=R.Tensor((512, 2048), dtype="float16"))
permute_dims23: R.Tensor((2048, 512), dtype="float16") = R.permute_dims(lv30, axes=None)
matmul23: R.Tensor((1, seq_len, 512), dtype="float32") = R.matmul(astype10, permute_dims23, out_dtype="float32")
add33: R.Tensor((1, seq_len, 512), dtype="float32") = R.add(matmul23, gpt_neox_layers_5_mlp_dense_4h_to_h_bias1)
astype11: R.Tensor((1, seq_len, 512), dtype="float16") = R.astype(add33, dtype="float16")
add34: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(astype11, add31)
add35: R.Tensor((1, seq_len, 512), dtype="float16") = R.add(add34, add29)
layer_norm12: R.Tensor((1, seq_len, 512), dtype="float16") = R.nn.layer_norm(add35, gpt_neox_final_layer_norm_weight1, gpt_neox_final_layer_norm_bias1, axes=[-1], epsilon=1.0000000000000001e-05, center=True, scale=True)
lv31 = R.call_tir(cls.index, (layer_norm12,), out_sinfo=R.Tensor((1, 1, 512), dtype="float16"))
lv32 = R.call_tir(cls.dequantize, (embed_out_q_weight1, embed_out_q_scale1), out_sinfo=R.Tensor((vocab_size, 512), dtype="float16"))
permute_dims24: R.Tensor((512, vocab_size), dtype="float16") = R.permute_dims(lv32, axes=None)
matmul24: R.Tensor((1, 1, vocab_size), dtype="float16") = R.matmul(lv31, permute_dims24, out_dtype="void")
astype12: R.Tensor((1, 1, vocab_size), dtype="float32") = R.astype(matmul24, dtype="float32")
gv1: R.Tuple(R.Tensor((1, 1, vocab_size), dtype="float32"), R.Object) = astype12, paged_kv_cache
R.output(gv1)
return gv1
@R.function
def softmax_with_temperature(logits: R.Tensor(("batch_size", 1, "vocab_size"), dtype="float32"), temperature: R.Tensor(("batch_size",), dtype="float32")) -> R.Tensor(("batch_size", 1, "vocab_size"), dtype="float32"):
batch_size = T.int64(is_size_var=True)
vocab_size = T.int64(is_size_var=True)
R.func_attr({"relax.memory_plan_dynamic_func_output": True, "tir_non_negative_var": ["vocab_size"], "tir_var_upper_bound": {"batch_size": 128, "seq_len": 2048, "total_seq_len": 2048}})
cls = Module
with R.dataflow():
lv: R.Tensor((batch_size, vocab_size), dtype="float32") = R.call_pure_packed("vm.builtin.reshape", logits, R.shape([batch_size, vocab_size]), sinfo_args=(R.Tensor((batch_size, vocab_size), dtype="float32"),))
lv1 = R.call_tir(cls.chunk_lse, (lv, temperature), out_sinfo=[R.Tensor((batch_size, (vocab_size + 4096 - 1) // 4096), dtype="float32"), R.Tensor((batch_size, (vocab_size + 4096 - 1) // 4096), dtype="float32")])
lv2: R.Tensor((batch_size, (vocab_size + 4096 - 1) // 4096), dtype="float32") = lv1[0]
lv3: R.Tensor((batch_size, (vocab_size + 4096 - 1) // 4096), dtype="float32") = lv1[1]
lv4 = R.call_tir(cls.softmax_with_chunked_sum, (lv, temperature, lv2, lv3), out_sinfo=R.Tensor((batch_size, vocab_size), dtype="float32"))
gv: R.Tensor((batch_size, 1, vocab_size), dtype="float32") = R.call_pure_packed("vm.builtin.reshape", lv4, R.shape([batch_size, 1, vocab_size]), sinfo_args=(R.Tensor((batch_size, 1, vocab_size), dtype="float32"),))
R.output(gv)
return gv
# from tvm.script import ir as I
# from tvm.script import tir as T
# from tvm.script import relax as R
class Module:
I.module_attrs({"system_lib_prefix": "gpt_neox_q4f16_1_"})
def apply_bitmask_inplace(var_logits: T.handle, var_seq_ids: T.handle, var_bitmask: T.handle):
T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": T.bool(True), "tir.noalias": T.bool(True)})
batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True)
logits = T.match_buffer(var_logits, (batch_size, vocab_size))
num_seq = T.int32(is_size_var=True)
seq_ids = T.match_buffer(var_seq_ids, (num_seq,), "int32")
bitmask = T.match_buffer(var_bitmask, (batch_size, (vocab_size + 31) // 32), "int32")
# with T.block("root"):
for fused_s_v_0 in T.thread_binding((num_seq * vocab_size + 255) // 256, thread="blockIdx.x"):
for fused_s_v_1 in T.thread_binding(256, thread="threadIdx.x"):
with T.block("block"):
vs = T.axis.spatial(num_seq, (fused_s_v_0 * 256 + fused_s_v_1) // vocab_size)
vv = T.axis.spatial(vocab_size, (fused_s_v_0 * 256 + fused_s_v_1) % vocab_size)
T.where(fused_s_v_0 * 256 + fused_s_v_1 < num_seq * vocab_size)
T.reads(bitmask[seq_ids[vs], vv // 32], seq_ids[vs], logits[seq_ids[vs], vv])
T.writes(logits[seq_ids[vs], vv])
logits[seq_ids[vs], vv] = T.if_then_else(T.bitwise_and(T.shift_right(bitmask[seq_ids[vs], vv // 32], vv % 32), 1) == 1, logits[seq_ids[vs], vv], T.float32(-340282346638528859811704183484516925440.0))
def apply_logit_bias_inplace(var_logits: T.handle, var_pos2seq_id: T.handle, var_token_ids: T.handle, var_logit_bias: T.handle):
T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": T.bool(True), "tir.noalias": T.bool(True)})
batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True)
logits = T.match_buffer(var_logits, (batch_size, vocab_size))
num_token = T.int32(is_size_var=True)
pos2seq_id = T.match_buffer(var_pos2seq_id, (num_token,), "int32")
token_ids = T.match_buffer(var_token_ids, (num_token,), "int32")
logit_bias = T.match_buffer(var_logit_bias, (num_token,))
# with T.block("root"):
for p0 in T.thread_binding((num_token + 255) // 256, thread="blockIdx.x"):
for p1 in T.thread_binding(256, thread="threadIdx.x"):
with T.block("block"):
vp = T.axis.spatial(num_token, p0 * 256 + p1)
T.where(p0 * 256 + p1 < num_token)
T.reads(logits[pos2seq_id[vp], token_ids[vp]], pos2seq_id[vp], token_ids[vp], logit_bias[vp])
T.writes(logits[pos2seq_id[vp], token_ids[vp]])
logits[pos2seq_id[vp], token_ids[vp]] = logits[pos2seq_id[vp], token_ids[vp]] + logit_bias[vp]
def apply_penalty_inplace(var_logits: T.handle, var_seq_ids: T.handle, var_pos2seq_id: T.handle, var_token_ids: T.handle, var_token_cnt: T.handle, var_penalties: T.handle):
T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": T.bool(True), "tir.noalias": T.bool(True)})
batch_size, vocab_size = T.int32(is_size_var=True), T.int32(is_size_var=True)
logits = T.match_buffer(var_logits, (batch_size, vocab_size))
num_seq = T.int32(is_size_var=True)
seq_ids = T.match_buffer(var_seq_ids, (num_seq,), "int32")
num_token = T.int32(is_size_var=True)
pos2seq_id = T.match_buffer(var_pos2seq_id, (num_token,), "int32")
token_ids = T.match_buffer(var_token_ids, (num_token,), "int32")
token_cnt = T.match_buffer(var_token_cnt, (num_token,), "int32")
penalties = T.match_buffer(var_penalties, (num_seq, 3))
# with T.block("root"):
for p0 in T.thread_binding((num_token + 255) // 256, thread="blockIdx.x"):
for p1 in T.thread_binding(256, thread="threadIdx.x"):
with T.block("block"):
vp = T.axis.spatial(num_token, p0 * 256 + p1)
T.where(p0 * 256 + p1 < num_token)
T.reads(logits[seq_ids[pos2seq_id[vp]], token_ids[vp]], seq_ids[pos2seq_id[vp]], pos2seq_id[vp], token_ids[vp], penalties[pos2seq_id[vp], 0:3], token_cnt[vp])
T.writes(logits[seq_ids[pos2seq_id[vp]], token_ids[vp]])
logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] = logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] - (penalties[pos2seq_id[vp], 0] + T.Cast("float32", token_cnt[vp]) * penalties[pos2seq_id[vp], 1])
logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] = T.if_then_else(logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] < T.float32(0.0), logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] * penalties[pos2seq_id[vp], 2], logits[seq_ids[pos2seq_id[vp]], token_ids[vp]] / penalties[pos2seq_id[vp], 2])
def batch_decode_paged_kv(_0: T.int32, Q_handle: T.handle, pages_handle: T.handle, page_table_indptr_handle: T.handle, page_table_values_handle: T.handle, var_length_info: T.handle, k_rope_pos_offset_handle: T.handle, q_rope_position_handle: T.handle, output_handle: T.handle, lse_handle: T.handle, rotary_mode: T.int32, rope_scale: T.float32, rope_theta: T.float32, attn_score_scaling_factor: T.float32):
T.func_attr({"target": T.target({"host": {"keys": ["arm_cpu", "cpu"], "kind": "llvm", "mtriple": "aarch64-linux-android", "tag": ""}, "keys": ["opencl", "gpu"], "kind": "opencl", "max_function_args": 128, "max_num_threads": 256, "max_shared_memory_per_block": 16384, "max_threads_per_block": 256, "tag": "", "texture_spatial_limit": 16384, "thread_warp_size": 1}), "tir.is_scheduled": 1})
B = T.int32(is_size_var=True)
Q = T.match_buffer(Q_handle, (B, 8, 64), "float16")
max_num_pages = T.int32(is_size_var=True)
pages = T.match_buffer(pages_handle, (max_num_pages, 2, 8, 16, 64), "float16", offset_factor=1)
page_table_indptr = T.match_buffer(page_table_indptr_handle, (B + 1,), "int32", offset_factor=1)
nnz_pages = T.int32(is_size_var=True)
page_table_values = T.match_buffer(page_table_values_handle, (nnz_pages,), "int32", offset_factor=1)
length_info = T.match_buffer(var_length_info, (B,), "int32", offset_factor=1)
k_rope_pos_offset = T.match_buffer(k_rope_pos_offset_handle, (B,), "int32", offset_factor=1)
q_rope_position = T.match_buffer(q_rope_position_handle, (B,), "int32", offset_factor=1)
output = T.match_buffer(output_handle, (B, 8, 64), "float16")
lse = T.match_buffer(lse_handle, (B, 8))
# with T.block("root"):
sm_scale: T.float32 = T.float32(0.18033688011112042)
for bx in T.thread_binding(B, thread="blockIdx.x"):
for fused_by_bz in T.thread_binding(8, thread="blockIdx.y"):
for ty in T.thread_binding(1, thread="threadIdx.y"):
for tx in T.thread_binding(16, thread="threadIdx.x"):
for tz in T.thread_binding(16, thread="threadIdx.z"):
with T.block("attn"):
T.reads(page_table_indptr[bx:bx + 2], length_info[bx], q_rope_position[bx], Q[bx, fused_by_bz // 8 + ty + fused_by_bz % 8, tx * 4 - 32:tx * 4 - 32 + 68])
T.writes(output[bx, fused_by_bz % 8 + fused_by_bz // 8 + ty, tx * 4:tx * 4 + 4], lse[bx, fused_by_bz % 8 + fused_by_bz // 8 + ty])
Q_local = T.alloc_buffer((4,), "float16", scope="local")
kv_chunk_len = T.alloc_buffer((1,), "int32", scope="local")
K_smem = T.alloc_buffer((16, 64), "float16", scope="shared")
V_smem = T.alloc_buffer((16, 64), "float16", scope="shared")
O_allreduce = T.alloc_buffer((16, 1, 64), scope="shared")
md_allreduce = T.alloc_buffer((16, 1, 2), scope="shared")
S_reduce_local = T.alloc_buffer((1,), scope="local")