diff --git "a/ir_dump/kernel.cl" "b/ir_dump/kernel.cl" new file mode 100644--- /dev/null +++ "b/ir_dump/kernel.cl" @@ -0,0 +1,11185 @@ +// Function: gpt_neox_q0f16_reshape2_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size) { + T_reshape[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] = lv73[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]; +} + +// Function: gpt_neox_q0f16_gather_probs_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n) { + if ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) < (batch_size * n)) { + dst[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] = src[((indices[(((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % (n * batch_size)) / n)] * n) + ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % n))]; + } +} + +// Function: gpt_neox_q0f16_reshape6_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len) { + T_reshape[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] = lv49[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]; +} + +// Function: gpt_neox_q0f16_merge_state_inplace_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N) { + float s_val[1]; + float s_other_val[1]; + float s_max[1]; + float scale[1]; + float other_scale[1]; + half v_vec[4]; + half v_other_vec[4]; + s_val[0] = S[((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_group_id(0))) * H)) + (convert_int(get_local_id(1))))]; + s_other_val[0] = S_other[((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_group_id(0))) * H)) + (convert_int(get_local_id(1))))]; + s_max[0] = max((float)s_val[0], (float)s_other_val[0]); + s_val[0] = exp2((s_val[0] - s_max[0])); + s_other_val[0] = exp2((s_other_val[0] - s_max[0])); + scale[0] = (s_val[0] / (s_val[0] + s_other_val[0])); + other_scale[0] = (s_other_val[0] / (s_val[0] + s_other_val[0])); + int4 v_ = (int4)(((((convert_int(get_local_id(0))) * 4) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_group_id(0))) * H)) + (convert_int(get_local_id(1)))) * D)))+(1*0), ((((convert_int(get_local_id(0))) * 4) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_group_id(0))) * H)) + (convert_int(get_local_id(1)))) * D)))+(1*1), ((((convert_int(get_local_id(0))) * 4) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_group_id(0))) * H)) + (convert_int(get_local_id(1)))) * D)))+(1*2), ((((convert_int(get_local_id(0))) * 4) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_group_id(0))) * H)) + (convert_int(get_local_id(1)))) * D)))+(1*3)); + vstore4(((half4)(V[v_.s0],V[v_.s1],V[v_.s2],V[v_.s3])), 0, v_vec + 0); + vstore4(((half4)(V_other[v_.s0],V_other[v_.s1],V_other[v_.s2],V_other[v_.s3])), 0, v_other_vec + 0); + for (int vec = 0; vec < 4; ++vec) { + v_vec[vec] = (convert_half((((convert_float(v_vec[vec])) * scale[0]) + ((convert_float(v_other_vec[vec])) * other_scale[0])))); + } + vstore4(vload4(0, v_vec + 0), 0, V + (((convert_int(get_local_id(0))) * 4) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_group_id(0))) * H)) + (convert_int(get_local_id(1)))) * D))); + S[((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_group_id(0))) * H)) + (convert_int(get_local_id(1))))] = (log2((s_val[0] + s_other_val[0])) + s_max[0]); +} + +// Function: gpt_neox_q0f16_index_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len) { + index[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] = layer_norm48[((((seq_len * 2048) + ((convert_int(get_group_id(0))) * 256)) + (convert_int(get_local_id(0)))) - 2048)]; +} + +// Function: gpt_neox_q0f16_fused_NT_matmul5_add5_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len) { + half NT_matmul_intermediate_pad_rf_local[32]; + half NT_matmul_intermediate_pad_rf_local_1[4]; + half NT_matmul_intermediate_pad_local[4]; + __local half red_buf0[64]; + for (int ax0_1_init = 0; ax0_1_init < 2; ++ax0_1_init) { + for (int ax1_fused_2_init = 0; ax1_fused_2_init < 2; ++ax1_fused_2_init) { + int8 v_ = (int8)((((ax0_1_init * 2) + ax1_fused_2_init))+(4*0), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*1), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*2), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*3), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*4), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*5), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*6), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*7)); + half8 v__1 = (half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f); + NT_matmul_intermediate_pad_rf_local[v_.s0] = v__1.s0; + NT_matmul_intermediate_pad_rf_local[v_.s1] = v__1.s1; + NT_matmul_intermediate_pad_rf_local[v_.s2] = v__1.s2; + NT_matmul_intermediate_pad_rf_local[v_.s3] = v__1.s3; + NT_matmul_intermediate_pad_rf_local[v_.s4] = v__1.s4; + NT_matmul_intermediate_pad_rf_local[v_.s5] = v__1.s5; + NT_matmul_intermediate_pad_rf_local[v_.s6] = v__1.s6; + NT_matmul_intermediate_pad_rf_local[v_.s7] = v__1.s7; + } + } + for (int ax2_fused_0 = 0; ax2_fused_0 < 8; ++ax2_fused_0) { + if (((convert_int(get_group_id(1))) * 2) < seq_len) { + int8 v__2 = (int8)((0)+(4*0), (0)+(4*1), (0)+(4*2), (0)+(4*3), (0)+(4*4), (0)+(4*5), (0)+(4*6), (0)+(4*7)); + half8 v__3 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__2.s0],NT_matmul_intermediate_pad_rf_local[v__2.s1],NT_matmul_intermediate_pad_rf_local[v__2.s2],NT_matmul_intermediate_pad_rf_local[v__2.s3],NT_matmul_intermediate_pad_rf_local[v__2.s4],NT_matmul_intermediate_pad_rf_local[v__2.s5],NT_matmul_intermediate_pad_rf_local[v__2.s6],NT_matmul_intermediate_pad_rf_local[v__2.s7])) + (vload8(0, layer_norm98 + ((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, gpt_neox_layers_0_attention_query_key_value_weight3 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))); + NT_matmul_intermediate_pad_rf_local[v__2.s0] = v__3.s0; + NT_matmul_intermediate_pad_rf_local[v__2.s1] = v__3.s1; + NT_matmul_intermediate_pad_rf_local[v__2.s2] = v__3.s2; + NT_matmul_intermediate_pad_rf_local[v__2.s3] = v__3.s3; + NT_matmul_intermediate_pad_rf_local[v__2.s4] = v__3.s4; + NT_matmul_intermediate_pad_rf_local[v__2.s5] = v__3.s5; + NT_matmul_intermediate_pad_rf_local[v__2.s6] = v__3.s6; + NT_matmul_intermediate_pad_rf_local[v__2.s7] = v__3.s7; + int8 v__4 = (int8)((1)+(4*0), (1)+(4*1), (1)+(4*2), (1)+(4*3), (1)+(4*4), (1)+(4*5), (1)+(4*6), (1)+(4*7)); + half8 v__5 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__4.s0],NT_matmul_intermediate_pad_rf_local[v__4.s1],NT_matmul_intermediate_pad_rf_local[v__4.s2],NT_matmul_intermediate_pad_rf_local[v__4.s3],NT_matmul_intermediate_pad_rf_local[v__4.s4],NT_matmul_intermediate_pad_rf_local[v__4.s5],NT_matmul_intermediate_pad_rf_local[v__4.s6],NT_matmul_intermediate_pad_rf_local[v__4.s7])) + (vload8(0, layer_norm98 + ((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, gpt_neox_layers_0_attention_query_key_value_weight3 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))); + NT_matmul_intermediate_pad_rf_local[v__4.s0] = v__5.s0; + NT_matmul_intermediate_pad_rf_local[v__4.s1] = v__5.s1; + NT_matmul_intermediate_pad_rf_local[v__4.s2] = v__5.s2; + NT_matmul_intermediate_pad_rf_local[v__4.s3] = v__5.s3; + NT_matmul_intermediate_pad_rf_local[v__4.s4] = v__5.s4; + NT_matmul_intermediate_pad_rf_local[v__4.s5] = v__5.s5; + NT_matmul_intermediate_pad_rf_local[v__4.s6] = v__5.s6; + NT_matmul_intermediate_pad_rf_local[v__4.s7] = v__5.s7; + } + if ((((convert_int(get_group_id(1))) * 2) + 1) < seq_len) { + int8 v__6 = (int8)((2)+(4*0), (2)+(4*1), (2)+(4*2), (2)+(4*3), (2)+(4*4), (2)+(4*5), (2)+(4*6), (2)+(4*7)); + half8 v__7 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__6.s0],NT_matmul_intermediate_pad_rf_local[v__6.s1],NT_matmul_intermediate_pad_rf_local[v__6.s2],NT_matmul_intermediate_pad_rf_local[v__6.s3],NT_matmul_intermediate_pad_rf_local[v__6.s4],NT_matmul_intermediate_pad_rf_local[v__6.s5],NT_matmul_intermediate_pad_rf_local[v__6.s6],NT_matmul_intermediate_pad_rf_local[v__6.s7])) + (vload8(0, layer_norm98 + (((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)) * vload8(0, gpt_neox_layers_0_attention_query_key_value_weight3 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))); + NT_matmul_intermediate_pad_rf_local[v__6.s0] = v__7.s0; + NT_matmul_intermediate_pad_rf_local[v__6.s1] = v__7.s1; + NT_matmul_intermediate_pad_rf_local[v__6.s2] = v__7.s2; + NT_matmul_intermediate_pad_rf_local[v__6.s3] = v__7.s3; + NT_matmul_intermediate_pad_rf_local[v__6.s4] = v__7.s4; + NT_matmul_intermediate_pad_rf_local[v__6.s5] = v__7.s5; + NT_matmul_intermediate_pad_rf_local[v__6.s6] = v__7.s6; + NT_matmul_intermediate_pad_rf_local[v__6.s7] = v__7.s7; + int8 v__8 = (int8)((3)+(4*0), (3)+(4*1), (3)+(4*2), (3)+(4*3), (3)+(4*4), (3)+(4*5), (3)+(4*6), (3)+(4*7)); + half8 v__9 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__8.s0],NT_matmul_intermediate_pad_rf_local[v__8.s1],NT_matmul_intermediate_pad_rf_local[v__8.s2],NT_matmul_intermediate_pad_rf_local[v__8.s3],NT_matmul_intermediate_pad_rf_local[v__8.s4],NT_matmul_intermediate_pad_rf_local[v__8.s5],NT_matmul_intermediate_pad_rf_local[v__8.s6],NT_matmul_intermediate_pad_rf_local[v__8.s7])) + (vload8(0, layer_norm98 + (((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)) * vload8(0, gpt_neox_layers_0_attention_query_key_value_weight3 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))); + NT_matmul_intermediate_pad_rf_local[v__8.s0] = v__9.s0; + NT_matmul_intermediate_pad_rf_local[v__8.s1] = v__9.s1; + NT_matmul_intermediate_pad_rf_local[v__8.s2] = v__9.s2; + NT_matmul_intermediate_pad_rf_local[v__8.s3] = v__9.s3; + NT_matmul_intermediate_pad_rf_local[v__8.s4] = v__9.s4; + NT_matmul_intermediate_pad_rf_local[v__8.s5] = v__9.s5; + NT_matmul_intermediate_pad_rf_local[v__8.s6] = v__9.s6; + NT_matmul_intermediate_pad_rf_local[v__8.s7] = v__9.s7; + } + } + for (int ax3_fused_2_0 = 0; ax3_fused_2_0 < 1; ++ax3_fused_2_0) { + for (int ax2 = 0; ax2 < 2; ++ax2) { + vstore2(((half2)((half)0.000000e+00f, (half)0.000000e+00f)), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + (ax2 * 2))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 4))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 8))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 12))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 16))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 20))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 24))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 28))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + } + } + for (int ax2_fused_2 = 0; ax2_fused_2 < 2; ++ax2_fused_2) { + for (int ax1 = 0; ax1 < 2; ++ax1) { + barrier(CLK_LOCAL_MEM_FENCE); + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_intermediate_pad_rf_local_1[((ax1 * 2) + ax2_fused_2)]; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 16) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 8) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 4) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 2) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + NT_matmul_intermediate_pad_local[((ax1 * 2) + ax2_fused_2)] = ((volatile __local half*)red_buf0)[((convert_int(get_local_id(0))) * 32)]; + } + } + for (int ax0 = 0; ax0 < 2; ++ax0) { + for (int ax1_fused_2 = 0; ax1_fused_2 < 2; ++ax1_fused_2) { + if (((bool)(((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)(((bool)(((convert_int(get_group_id(1))) - ((seq_len + 1) >> 1)) < 0)) || ((bool)((convert_int(get_group_id(1))) == 0)))))) && ((bool)((((convert_int(get_group_id(1))) * 2) + ax0) < seq_len))) { + T_add[((((((convert_int(get_group_id(1))) * 12288) + (ax0 * 6144)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] = (NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_attention_query_key_value_bias3[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]); + } + } + } +} + +// Function: gpt_neox_q0f16_fused_rope_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len) { + if (((convert_int(get_group_id(0))) % 24) < 8) { + half condval; + if ((((bool)(0 < apply_rope)) && ((bool)(((convert_int(get_local_id(0))) & 127) < 32)))) { + float freq = ((convert_float(position_map[(((convert_int(get_group_id(0))) / 24) + position_map_elem_offset)])) / pow(1.000000e+04f, ((convert_float((((convert_int(get_local_id(0))) & 15) * 2))) * 3.125000e-02f))); + half condval_1; + if ((((convert_int(get_local_id(0))) & 127) < 16)) { + condval_1 = (qkv[((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) + 16)] * (half)-1.000000e+00f); + } else { + condval_1 = qkv[((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) - 16)]; + } + condval = (convert_half(((cos(freq) * (convert_float(qkv[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]))) + (sin(freq) * (convert_float(condval_1)))))); + } else { + condval = qkv[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]; + } + q[(((((convert_int(get_group_id(0))) / 24) * 2048) + (((convert_int(get_group_id(0))) % 24) * 256)) + (convert_int(get_local_id(0))))] = condval; + } else { + if (((convert_int(get_group_id(0))) % 24) < 16) { + half condval_2; + if ((((bool)(0 < apply_rope)) && ((bool)(((convert_int(get_local_id(0))) & 127) < 32)))) { + float freq_1 = ((convert_float(position_map[(((convert_int(get_group_id(0))) / 24) + position_map_elem_offset)])) / pow(1.000000e+04f, ((convert_float((((convert_int(get_local_id(0))) & 15) * 2))) * 3.125000e-02f))); + half condval_3; + if ((((convert_int(get_local_id(0))) & 127) < 16)) { + condval_3 = (qkv[((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) + 16)] * (half)-1.000000e+00f); + } else { + condval_3 = qkv[((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) - 16)]; + } + condval_2 = (convert_half(((cos(freq_1) * (convert_float(qkv[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]))) + (sin(freq_1) * (convert_float(condval_3)))))); + } else { + condval_2 = qkv[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]; + } + k[((((((convert_int(get_group_id(0))) / 24) * 2048) + (((convert_int(get_group_id(0))) % 24) * 256)) + (convert_int(get_local_id(0)))) - 2048)] = condval_2; + } else { + v[((((((convert_int(get_group_id(0))) / 24) * 2048) + (((convert_int(get_group_id(0))) % 24) * 256)) + (convert_int(get_local_id(0)))) - 4096)] = qkv[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]; + } + } +} + +// Function: gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1 +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len) { + float NT_matmul_intermediate_pad_rf_local[64]; + float NT_matmul_intermediate_pad_rf_local_1[8]; + float NT_matmul_intermediate_pad_local[8]; + __local float red_buf0[64]; + for (int ax0_1_init = 0; ax0_1_init < 4; ++ax0_1_init) { + for (int ax1_fused_2_init = 0; ax1_fused_2_init < 2; ++ax1_fused_2_init) { + int8 v_ = (int8)((((ax0_1_init * 2) + ax1_fused_2_init))+(8*0), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*1), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*2), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*3), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*4), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*5), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*6), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*7)); + float8 v__1 = (float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f); + NT_matmul_intermediate_pad_rf_local[v_.s0] = v__1.s0; + NT_matmul_intermediate_pad_rf_local[v_.s1] = v__1.s1; + NT_matmul_intermediate_pad_rf_local[v_.s2] = v__1.s2; + NT_matmul_intermediate_pad_rf_local[v_.s3] = v__1.s3; + NT_matmul_intermediate_pad_rf_local[v_.s4] = v__1.s4; + NT_matmul_intermediate_pad_rf_local[v_.s5] = v__1.s5; + NT_matmul_intermediate_pad_rf_local[v_.s6] = v__1.s6; + NT_matmul_intermediate_pad_rf_local[v_.s7] = v__1.s7; + } + } + for (int ax2_fused_0 = 0; ax2_fused_0 < 8; ++ax2_fused_0) { + if (((convert_int(get_group_id(1))) * 4) < seq_len) { + int8 v__2 = (int8)((0)+(8*0), (0)+(8*1), (0)+(8*2), (0)+(8*3), (0)+(8*4), (0)+(8*5), (0)+(8*6), (0)+(8*7)); + float8 v__3 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__2.s0],NT_matmul_intermediate_pad_rf_local[v__2.s1],NT_matmul_intermediate_pad_rf_local[v__2.s2],NT_matmul_intermediate_pad_rf_local[v__2.s3],NT_matmul_intermediate_pad_rf_local[v__2.s4],NT_matmul_intermediate_pad_rf_local[v__2.s5],NT_matmul_intermediate_pad_rf_local[v__2.s6],NT_matmul_intermediate_pad_rf_local[v__2.s7])) + ((convert_float8(vload8(0, layer_norm99 + ((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_h_to_4h_weight3 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))))); + NT_matmul_intermediate_pad_rf_local[v__2.s0] = v__3.s0; + NT_matmul_intermediate_pad_rf_local[v__2.s1] = v__3.s1; + NT_matmul_intermediate_pad_rf_local[v__2.s2] = v__3.s2; + NT_matmul_intermediate_pad_rf_local[v__2.s3] = v__3.s3; + NT_matmul_intermediate_pad_rf_local[v__2.s4] = v__3.s4; + NT_matmul_intermediate_pad_rf_local[v__2.s5] = v__3.s5; + NT_matmul_intermediate_pad_rf_local[v__2.s6] = v__3.s6; + NT_matmul_intermediate_pad_rf_local[v__2.s7] = v__3.s7; + int8 v__4 = (int8)((1)+(8*0), (1)+(8*1), (1)+(8*2), (1)+(8*3), (1)+(8*4), (1)+(8*5), (1)+(8*6), (1)+(8*7)); + float8 v__5 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__4.s0],NT_matmul_intermediate_pad_rf_local[v__4.s1],NT_matmul_intermediate_pad_rf_local[v__4.s2],NT_matmul_intermediate_pad_rf_local[v__4.s3],NT_matmul_intermediate_pad_rf_local[v__4.s4],NT_matmul_intermediate_pad_rf_local[v__4.s5],NT_matmul_intermediate_pad_rf_local[v__4.s6],NT_matmul_intermediate_pad_rf_local[v__4.s7])) + ((convert_float8(vload8(0, layer_norm99 + ((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_h_to_4h_weight3 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))))); + NT_matmul_intermediate_pad_rf_local[v__4.s0] = v__5.s0; + NT_matmul_intermediate_pad_rf_local[v__4.s1] = v__5.s1; + NT_matmul_intermediate_pad_rf_local[v__4.s2] = v__5.s2; + NT_matmul_intermediate_pad_rf_local[v__4.s3] = v__5.s3; + NT_matmul_intermediate_pad_rf_local[v__4.s4] = v__5.s4; + NT_matmul_intermediate_pad_rf_local[v__4.s5] = v__5.s5; + NT_matmul_intermediate_pad_rf_local[v__4.s6] = v__5.s6; + NT_matmul_intermediate_pad_rf_local[v__4.s7] = v__5.s7; + } + if ((((convert_int(get_group_id(1))) * 4) + 1) < seq_len) { + int8 v__6 = (int8)((2)+(8*0), (2)+(8*1), (2)+(8*2), (2)+(8*3), (2)+(8*4), (2)+(8*5), (2)+(8*6), (2)+(8*7)); + float8 v__7 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__6.s0],NT_matmul_intermediate_pad_rf_local[v__6.s1],NT_matmul_intermediate_pad_rf_local[v__6.s2],NT_matmul_intermediate_pad_rf_local[v__6.s3],NT_matmul_intermediate_pad_rf_local[v__6.s4],NT_matmul_intermediate_pad_rf_local[v__6.s5],NT_matmul_intermediate_pad_rf_local[v__6.s6],NT_matmul_intermediate_pad_rf_local[v__6.s7])) + ((convert_float8(vload8(0, layer_norm99 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_h_to_4h_weight3 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))))); + NT_matmul_intermediate_pad_rf_local[v__6.s0] = v__7.s0; + NT_matmul_intermediate_pad_rf_local[v__6.s1] = v__7.s1; + NT_matmul_intermediate_pad_rf_local[v__6.s2] = v__7.s2; + NT_matmul_intermediate_pad_rf_local[v__6.s3] = v__7.s3; + NT_matmul_intermediate_pad_rf_local[v__6.s4] = v__7.s4; + NT_matmul_intermediate_pad_rf_local[v__6.s5] = v__7.s5; + NT_matmul_intermediate_pad_rf_local[v__6.s6] = v__7.s6; + NT_matmul_intermediate_pad_rf_local[v__6.s7] = v__7.s7; + int8 v__8 = (int8)((3)+(8*0), (3)+(8*1), (3)+(8*2), (3)+(8*3), (3)+(8*4), (3)+(8*5), (3)+(8*6), (3)+(8*7)); + float8 v__9 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__8.s0],NT_matmul_intermediate_pad_rf_local[v__8.s1],NT_matmul_intermediate_pad_rf_local[v__8.s2],NT_matmul_intermediate_pad_rf_local[v__8.s3],NT_matmul_intermediate_pad_rf_local[v__8.s4],NT_matmul_intermediate_pad_rf_local[v__8.s5],NT_matmul_intermediate_pad_rf_local[v__8.s6],NT_matmul_intermediate_pad_rf_local[v__8.s7])) + ((convert_float8(vload8(0, layer_norm99 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_h_to_4h_weight3 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))))); + NT_matmul_intermediate_pad_rf_local[v__8.s0] = v__9.s0; + NT_matmul_intermediate_pad_rf_local[v__8.s1] = v__9.s1; + NT_matmul_intermediate_pad_rf_local[v__8.s2] = v__9.s2; + NT_matmul_intermediate_pad_rf_local[v__8.s3] = v__9.s3; + NT_matmul_intermediate_pad_rf_local[v__8.s4] = v__9.s4; + NT_matmul_intermediate_pad_rf_local[v__8.s5] = v__9.s5; + NT_matmul_intermediate_pad_rf_local[v__8.s6] = v__9.s6; + NT_matmul_intermediate_pad_rf_local[v__8.s7] = v__9.s7; + } + if ((((convert_int(get_group_id(1))) * 4) + 2) < seq_len) { + int8 v__10 = (int8)((4)+(8*0), (4)+(8*1), (4)+(8*2), (4)+(8*3), (4)+(8*4), (4)+(8*5), (4)+(8*6), (4)+(8*7)); + float8 v__11 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__10.s0],NT_matmul_intermediate_pad_rf_local[v__10.s1],NT_matmul_intermediate_pad_rf_local[v__10.s2],NT_matmul_intermediate_pad_rf_local[v__10.s3],NT_matmul_intermediate_pad_rf_local[v__10.s4],NT_matmul_intermediate_pad_rf_local[v__10.s5],NT_matmul_intermediate_pad_rf_local[v__10.s6],NT_matmul_intermediate_pad_rf_local[v__10.s7])) + ((convert_float8(vload8(0, layer_norm99 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 4096)))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_h_to_4h_weight3 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))))); + NT_matmul_intermediate_pad_rf_local[v__10.s0] = v__11.s0; + NT_matmul_intermediate_pad_rf_local[v__10.s1] = v__11.s1; + NT_matmul_intermediate_pad_rf_local[v__10.s2] = v__11.s2; + NT_matmul_intermediate_pad_rf_local[v__10.s3] = v__11.s3; + NT_matmul_intermediate_pad_rf_local[v__10.s4] = v__11.s4; + NT_matmul_intermediate_pad_rf_local[v__10.s5] = v__11.s5; + NT_matmul_intermediate_pad_rf_local[v__10.s6] = v__11.s6; + NT_matmul_intermediate_pad_rf_local[v__10.s7] = v__11.s7; + int8 v__12 = (int8)((5)+(8*0), (5)+(8*1), (5)+(8*2), (5)+(8*3), (5)+(8*4), (5)+(8*5), (5)+(8*6), (5)+(8*7)); + float8 v__13 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__12.s0],NT_matmul_intermediate_pad_rf_local[v__12.s1],NT_matmul_intermediate_pad_rf_local[v__12.s2],NT_matmul_intermediate_pad_rf_local[v__12.s3],NT_matmul_intermediate_pad_rf_local[v__12.s4],NT_matmul_intermediate_pad_rf_local[v__12.s5],NT_matmul_intermediate_pad_rf_local[v__12.s6],NT_matmul_intermediate_pad_rf_local[v__12.s7])) + ((convert_float8(vload8(0, layer_norm99 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 4096)))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_h_to_4h_weight3 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))))); + NT_matmul_intermediate_pad_rf_local[v__12.s0] = v__13.s0; + NT_matmul_intermediate_pad_rf_local[v__12.s1] = v__13.s1; + NT_matmul_intermediate_pad_rf_local[v__12.s2] = v__13.s2; + NT_matmul_intermediate_pad_rf_local[v__12.s3] = v__13.s3; + NT_matmul_intermediate_pad_rf_local[v__12.s4] = v__13.s4; + NT_matmul_intermediate_pad_rf_local[v__12.s5] = v__13.s5; + NT_matmul_intermediate_pad_rf_local[v__12.s6] = v__13.s6; + NT_matmul_intermediate_pad_rf_local[v__12.s7] = v__13.s7; + } + if ((((convert_int(get_group_id(1))) * 4) + 3) < seq_len) { + int8 v__14 = (int8)((6)+(8*0), (6)+(8*1), (6)+(8*2), (6)+(8*3), (6)+(8*4), (6)+(8*5), (6)+(8*6), (6)+(8*7)); + float8 v__15 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__14.s0],NT_matmul_intermediate_pad_rf_local[v__14.s1],NT_matmul_intermediate_pad_rf_local[v__14.s2],NT_matmul_intermediate_pad_rf_local[v__14.s3],NT_matmul_intermediate_pad_rf_local[v__14.s4],NT_matmul_intermediate_pad_rf_local[v__14.s5],NT_matmul_intermediate_pad_rf_local[v__14.s6],NT_matmul_intermediate_pad_rf_local[v__14.s7])) + ((convert_float8(vload8(0, layer_norm99 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 6144)))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_h_to_4h_weight3 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))))); + NT_matmul_intermediate_pad_rf_local[v__14.s0] = v__15.s0; + NT_matmul_intermediate_pad_rf_local[v__14.s1] = v__15.s1; + NT_matmul_intermediate_pad_rf_local[v__14.s2] = v__15.s2; + NT_matmul_intermediate_pad_rf_local[v__14.s3] = v__15.s3; + NT_matmul_intermediate_pad_rf_local[v__14.s4] = v__15.s4; + NT_matmul_intermediate_pad_rf_local[v__14.s5] = v__15.s5; + NT_matmul_intermediate_pad_rf_local[v__14.s6] = v__15.s6; + NT_matmul_intermediate_pad_rf_local[v__14.s7] = v__15.s7; + int8 v__16 = (int8)((7)+(8*0), (7)+(8*1), (7)+(8*2), (7)+(8*3), (7)+(8*4), (7)+(8*5), (7)+(8*6), (7)+(8*7)); + float8 v__17 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__16.s0],NT_matmul_intermediate_pad_rf_local[v__16.s1],NT_matmul_intermediate_pad_rf_local[v__16.s2],NT_matmul_intermediate_pad_rf_local[v__16.s3],NT_matmul_intermediate_pad_rf_local[v__16.s4],NT_matmul_intermediate_pad_rf_local[v__16.s5],NT_matmul_intermediate_pad_rf_local[v__16.s6],NT_matmul_intermediate_pad_rf_local[v__16.s7])) + ((convert_float8(vload8(0, layer_norm99 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 6144)))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_h_to_4h_weight3 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))))); + NT_matmul_intermediate_pad_rf_local[v__16.s0] = v__17.s0; + NT_matmul_intermediate_pad_rf_local[v__16.s1] = v__17.s1; + NT_matmul_intermediate_pad_rf_local[v__16.s2] = v__17.s2; + NT_matmul_intermediate_pad_rf_local[v__16.s3] = v__17.s3; + NT_matmul_intermediate_pad_rf_local[v__16.s4] = v__17.s4; + NT_matmul_intermediate_pad_rf_local[v__16.s5] = v__17.s5; + NT_matmul_intermediate_pad_rf_local[v__16.s6] = v__17.s6; + NT_matmul_intermediate_pad_rf_local[v__16.s7] = v__17.s7; + } + } + for (int ax3_fused_2_0 = 0; ax3_fused_2_0 < 1; ++ax3_fused_2_0) { + for (int ax2 = 0; ax2 < 4; ++ax2) { + vstore2(((float2)(0.000000e+00f, 0.000000e+00f)), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + (ax2 * 2))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 8))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 16))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 24))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 32))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 40))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 48))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 56))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + } + } + for (int ax2_fused_2 = 0; ax2_fused_2 < 2; ++ax2_fused_2) { + for (int ax1 = 0; ax1 < 4; ++ax1) { + barrier(CLK_LOCAL_MEM_FENCE); + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_intermediate_pad_rf_local_1[((ax1 * 2) + ax2_fused_2)]; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 16) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 8) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 4) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 2) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + NT_matmul_intermediate_pad_local[((ax1 * 2) + ax2_fused_2)] = ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) * 32)]; + } + } + for (int ax0 = 0; ax0 < 4; ++ax0) { + for (int ax1_fused_2 = 0; ax1_fused_2 < 2; ++ax1_fused_2) { + if (((bool)(((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)(((bool)(((convert_int(get_group_id(1))) - ((seq_len + 3) >> 2)) < 0)) || ((bool)((convert_int(get_group_id(1))) == 0)))))) && ((bool)((((convert_int(get_group_id(1))) * 4) + ax0) < seq_len))) { + compute[((((((convert_int(get_group_id(1))) * 32768) + (ax0 * 8192)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] = (convert_half(((NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]) * 7.071068e-01f)) * 5.000000e-01f))))); + } + } + } +} + +// Function: gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1 +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size) { + half NT_matmul_intermediate_pad_rf_local[64]; + half NT_matmul_intermediate_pad_rf_local_1[8]; + half NT_matmul_intermediate_pad_local[8]; + __local half red_buf0[64]; + for (int ax0_1_init = 0; ax0_1_init < 4; ++ax0_1_init) { + for (int ax1_fused_2_init = 0; ax1_fused_2_init < 2; ++ax1_fused_2_init) { + int8 v_ = (int8)((((ax0_1_init * 2) + ax1_fused_2_init))+(8*0), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*1), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*2), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*3), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*4), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*5), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*6), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*7)); + half8 v__1 = (half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f); + NT_matmul_intermediate_pad_rf_local[v_.s0] = v__1.s0; + NT_matmul_intermediate_pad_rf_local[v_.s1] = v__1.s1; + NT_matmul_intermediate_pad_rf_local[v_.s2] = v__1.s2; + NT_matmul_intermediate_pad_rf_local[v_.s3] = v__1.s3; + NT_matmul_intermediate_pad_rf_local[v_.s4] = v__1.s4; + NT_matmul_intermediate_pad_rf_local[v_.s5] = v__1.s5; + NT_matmul_intermediate_pad_rf_local[v_.s6] = v__1.s6; + NT_matmul_intermediate_pad_rf_local[v_.s7] = v__1.s7; + } + } + for (int ax2_fused_0 = 0; ax2_fused_0 < 8; ++ax2_fused_0) { + if (((convert_int(get_group_id(1))) * 4) < batch_size) { + int8 v__2 = (int8)((0)+(8*0), (0)+(8*1), (0)+(8*2), (0)+(8*3), (0)+(8*4), (0)+(8*5), (0)+(8*6), (0)+(8*7)); + half8 v__3 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__2.s0],NT_matmul_intermediate_pad_rf_local[v__2.s1],NT_matmul_intermediate_pad_rf_local[v__2.s2],NT_matmul_intermediate_pad_rf_local[v__2.s3],NT_matmul_intermediate_pad_rf_local[v__2.s4],NT_matmul_intermediate_pad_rf_local[v__2.s5],NT_matmul_intermediate_pad_rf_local[v__2.s6],NT_matmul_intermediate_pad_rf_local[v__2.s7])) + (vload8(0, reshape291 + ((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, gpt_neox_layers_0_attention_dense_weight4 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))); + NT_matmul_intermediate_pad_rf_local[v__2.s0] = v__3.s0; + NT_matmul_intermediate_pad_rf_local[v__2.s1] = v__3.s1; + NT_matmul_intermediate_pad_rf_local[v__2.s2] = v__3.s2; + NT_matmul_intermediate_pad_rf_local[v__2.s3] = v__3.s3; + NT_matmul_intermediate_pad_rf_local[v__2.s4] = v__3.s4; + NT_matmul_intermediate_pad_rf_local[v__2.s5] = v__3.s5; + NT_matmul_intermediate_pad_rf_local[v__2.s6] = v__3.s6; + NT_matmul_intermediate_pad_rf_local[v__2.s7] = v__3.s7; + int8 v__4 = (int8)((1)+(8*0), (1)+(8*1), (1)+(8*2), (1)+(8*3), (1)+(8*4), (1)+(8*5), (1)+(8*6), (1)+(8*7)); + half8 v__5 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__4.s0],NT_matmul_intermediate_pad_rf_local[v__4.s1],NT_matmul_intermediate_pad_rf_local[v__4.s2],NT_matmul_intermediate_pad_rf_local[v__4.s3],NT_matmul_intermediate_pad_rf_local[v__4.s4],NT_matmul_intermediate_pad_rf_local[v__4.s5],NT_matmul_intermediate_pad_rf_local[v__4.s6],NT_matmul_intermediate_pad_rf_local[v__4.s7])) + (vload8(0, reshape291 + ((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, gpt_neox_layers_0_attention_dense_weight4 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))); + NT_matmul_intermediate_pad_rf_local[v__4.s0] = v__5.s0; + NT_matmul_intermediate_pad_rf_local[v__4.s1] = v__5.s1; + NT_matmul_intermediate_pad_rf_local[v__4.s2] = v__5.s2; + NT_matmul_intermediate_pad_rf_local[v__4.s3] = v__5.s3; + NT_matmul_intermediate_pad_rf_local[v__4.s4] = v__5.s4; + NT_matmul_intermediate_pad_rf_local[v__4.s5] = v__5.s5; + NT_matmul_intermediate_pad_rf_local[v__4.s6] = v__5.s6; + NT_matmul_intermediate_pad_rf_local[v__4.s7] = v__5.s7; + } + if ((((convert_int(get_group_id(1))) * 4) + 1) < batch_size) { + int8 v__6 = (int8)((2)+(8*0), (2)+(8*1), (2)+(8*2), (2)+(8*3), (2)+(8*4), (2)+(8*5), (2)+(8*6), (2)+(8*7)); + half8 v__7 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__6.s0],NT_matmul_intermediate_pad_rf_local[v__6.s1],NT_matmul_intermediate_pad_rf_local[v__6.s2],NT_matmul_intermediate_pad_rf_local[v__6.s3],NT_matmul_intermediate_pad_rf_local[v__6.s4],NT_matmul_intermediate_pad_rf_local[v__6.s5],NT_matmul_intermediate_pad_rf_local[v__6.s6],NT_matmul_intermediate_pad_rf_local[v__6.s7])) + (vload8(0, reshape291 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)) * vload8(0, gpt_neox_layers_0_attention_dense_weight4 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))); + NT_matmul_intermediate_pad_rf_local[v__6.s0] = v__7.s0; + NT_matmul_intermediate_pad_rf_local[v__6.s1] = v__7.s1; + NT_matmul_intermediate_pad_rf_local[v__6.s2] = v__7.s2; + NT_matmul_intermediate_pad_rf_local[v__6.s3] = v__7.s3; + NT_matmul_intermediate_pad_rf_local[v__6.s4] = v__7.s4; + NT_matmul_intermediate_pad_rf_local[v__6.s5] = v__7.s5; + NT_matmul_intermediate_pad_rf_local[v__6.s6] = v__7.s6; + NT_matmul_intermediate_pad_rf_local[v__6.s7] = v__7.s7; + int8 v__8 = (int8)((3)+(8*0), (3)+(8*1), (3)+(8*2), (3)+(8*3), (3)+(8*4), (3)+(8*5), (3)+(8*6), (3)+(8*7)); + half8 v__9 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__8.s0],NT_matmul_intermediate_pad_rf_local[v__8.s1],NT_matmul_intermediate_pad_rf_local[v__8.s2],NT_matmul_intermediate_pad_rf_local[v__8.s3],NT_matmul_intermediate_pad_rf_local[v__8.s4],NT_matmul_intermediate_pad_rf_local[v__8.s5],NT_matmul_intermediate_pad_rf_local[v__8.s6],NT_matmul_intermediate_pad_rf_local[v__8.s7])) + (vload8(0, reshape291 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)) * vload8(0, gpt_neox_layers_0_attention_dense_weight4 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))); + NT_matmul_intermediate_pad_rf_local[v__8.s0] = v__9.s0; + NT_matmul_intermediate_pad_rf_local[v__8.s1] = v__9.s1; + NT_matmul_intermediate_pad_rf_local[v__8.s2] = v__9.s2; + NT_matmul_intermediate_pad_rf_local[v__8.s3] = v__9.s3; + NT_matmul_intermediate_pad_rf_local[v__8.s4] = v__9.s4; + NT_matmul_intermediate_pad_rf_local[v__8.s5] = v__9.s5; + NT_matmul_intermediate_pad_rf_local[v__8.s6] = v__9.s6; + NT_matmul_intermediate_pad_rf_local[v__8.s7] = v__9.s7; + } + if ((((convert_int(get_group_id(1))) * 4) + 2) < batch_size) { + int8 v__10 = (int8)((4)+(8*0), (4)+(8*1), (4)+(8*2), (4)+(8*3), (4)+(8*4), (4)+(8*5), (4)+(8*6), (4)+(8*7)); + half8 v__11 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__10.s0],NT_matmul_intermediate_pad_rf_local[v__10.s1],NT_matmul_intermediate_pad_rf_local[v__10.s2],NT_matmul_intermediate_pad_rf_local[v__10.s3],NT_matmul_intermediate_pad_rf_local[v__10.s4],NT_matmul_intermediate_pad_rf_local[v__10.s5],NT_matmul_intermediate_pad_rf_local[v__10.s6],NT_matmul_intermediate_pad_rf_local[v__10.s7])) + (vload8(0, reshape291 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 4096)) * vload8(0, gpt_neox_layers_0_attention_dense_weight4 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))); + NT_matmul_intermediate_pad_rf_local[v__10.s0] = v__11.s0; + NT_matmul_intermediate_pad_rf_local[v__10.s1] = v__11.s1; + NT_matmul_intermediate_pad_rf_local[v__10.s2] = v__11.s2; + NT_matmul_intermediate_pad_rf_local[v__10.s3] = v__11.s3; + NT_matmul_intermediate_pad_rf_local[v__10.s4] = v__11.s4; + NT_matmul_intermediate_pad_rf_local[v__10.s5] = v__11.s5; + NT_matmul_intermediate_pad_rf_local[v__10.s6] = v__11.s6; + NT_matmul_intermediate_pad_rf_local[v__10.s7] = v__11.s7; + int8 v__12 = (int8)((5)+(8*0), (5)+(8*1), (5)+(8*2), (5)+(8*3), (5)+(8*4), (5)+(8*5), (5)+(8*6), (5)+(8*7)); + half8 v__13 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__12.s0],NT_matmul_intermediate_pad_rf_local[v__12.s1],NT_matmul_intermediate_pad_rf_local[v__12.s2],NT_matmul_intermediate_pad_rf_local[v__12.s3],NT_matmul_intermediate_pad_rf_local[v__12.s4],NT_matmul_intermediate_pad_rf_local[v__12.s5],NT_matmul_intermediate_pad_rf_local[v__12.s6],NT_matmul_intermediate_pad_rf_local[v__12.s7])) + (vload8(0, reshape291 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 4096)) * vload8(0, gpt_neox_layers_0_attention_dense_weight4 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))); + NT_matmul_intermediate_pad_rf_local[v__12.s0] = v__13.s0; + NT_matmul_intermediate_pad_rf_local[v__12.s1] = v__13.s1; + NT_matmul_intermediate_pad_rf_local[v__12.s2] = v__13.s2; + NT_matmul_intermediate_pad_rf_local[v__12.s3] = v__13.s3; + NT_matmul_intermediate_pad_rf_local[v__12.s4] = v__13.s4; + NT_matmul_intermediate_pad_rf_local[v__12.s5] = v__13.s5; + NT_matmul_intermediate_pad_rf_local[v__12.s6] = v__13.s6; + NT_matmul_intermediate_pad_rf_local[v__12.s7] = v__13.s7; + } + if ((((convert_int(get_group_id(1))) * 4) + 3) < batch_size) { + int8 v__14 = (int8)((6)+(8*0), (6)+(8*1), (6)+(8*2), (6)+(8*3), (6)+(8*4), (6)+(8*5), (6)+(8*6), (6)+(8*7)); + half8 v__15 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__14.s0],NT_matmul_intermediate_pad_rf_local[v__14.s1],NT_matmul_intermediate_pad_rf_local[v__14.s2],NT_matmul_intermediate_pad_rf_local[v__14.s3],NT_matmul_intermediate_pad_rf_local[v__14.s4],NT_matmul_intermediate_pad_rf_local[v__14.s5],NT_matmul_intermediate_pad_rf_local[v__14.s6],NT_matmul_intermediate_pad_rf_local[v__14.s7])) + (vload8(0, reshape291 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 6144)) * vload8(0, gpt_neox_layers_0_attention_dense_weight4 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))); + NT_matmul_intermediate_pad_rf_local[v__14.s0] = v__15.s0; + NT_matmul_intermediate_pad_rf_local[v__14.s1] = v__15.s1; + NT_matmul_intermediate_pad_rf_local[v__14.s2] = v__15.s2; + NT_matmul_intermediate_pad_rf_local[v__14.s3] = v__15.s3; + NT_matmul_intermediate_pad_rf_local[v__14.s4] = v__15.s4; + NT_matmul_intermediate_pad_rf_local[v__14.s5] = v__15.s5; + NT_matmul_intermediate_pad_rf_local[v__14.s6] = v__15.s6; + NT_matmul_intermediate_pad_rf_local[v__14.s7] = v__15.s7; + int8 v__16 = (int8)((7)+(8*0), (7)+(8*1), (7)+(8*2), (7)+(8*3), (7)+(8*4), (7)+(8*5), (7)+(8*6), (7)+(8*7)); + half8 v__17 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__16.s0],NT_matmul_intermediate_pad_rf_local[v__16.s1],NT_matmul_intermediate_pad_rf_local[v__16.s2],NT_matmul_intermediate_pad_rf_local[v__16.s3],NT_matmul_intermediate_pad_rf_local[v__16.s4],NT_matmul_intermediate_pad_rf_local[v__16.s5],NT_matmul_intermediate_pad_rf_local[v__16.s6],NT_matmul_intermediate_pad_rf_local[v__16.s7])) + (vload8(0, reshape291 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 6144)) * vload8(0, gpt_neox_layers_0_attention_dense_weight4 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))); + NT_matmul_intermediate_pad_rf_local[v__16.s0] = v__17.s0; + NT_matmul_intermediate_pad_rf_local[v__16.s1] = v__17.s1; + NT_matmul_intermediate_pad_rf_local[v__16.s2] = v__17.s2; + NT_matmul_intermediate_pad_rf_local[v__16.s3] = v__17.s3; + NT_matmul_intermediate_pad_rf_local[v__16.s4] = v__17.s4; + NT_matmul_intermediate_pad_rf_local[v__16.s5] = v__17.s5; + NT_matmul_intermediate_pad_rf_local[v__16.s6] = v__17.s6; + NT_matmul_intermediate_pad_rf_local[v__16.s7] = v__17.s7; + } + } + for (int ax3_fused_2_0 = 0; ax3_fused_2_0 < 1; ++ax3_fused_2_0) { + for (int ax2 = 0; ax2 < 4; ++ax2) { + vstore2(((half2)((half)0.000000e+00f, (half)0.000000e+00f)), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + (ax2 * 2))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 8))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 16))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 24))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 32))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 40))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 48))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 56))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + } + } + for (int ax2_fused_2 = 0; ax2_fused_2 < 2; ++ax2_fused_2) { + for (int ax1 = 0; ax1 < 4; ++ax1) { + barrier(CLK_LOCAL_MEM_FENCE); + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_intermediate_pad_rf_local_1[((ax1 * 2) + ax2_fused_2)]; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 16) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 8) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 4) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 2) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + NT_matmul_intermediate_pad_local[((ax1 * 2) + ax2_fused_2)] = ((volatile __local half*)red_buf0)[((convert_int(get_local_id(0))) * 32)]; + } + } + for (int ax0 = 0; ax0 < 4; ++ax0) { + for (int ax1_fused_2 = 0; ax1_fused_2 < 2; ++ax1_fused_2) { + if (((bool)(((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)(((bool)(((convert_int(get_group_id(1))) - ((batch_size + 3) >> 2)) < 0)) || ((bool)((convert_int(get_group_id(1))) == 0)))))) && ((bool)((((convert_int(get_group_id(1))) * 4) + ax0) < batch_size))) { + T_add[((((((convert_int(get_group_id(1))) * 8192) + (ax0 * 2048)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] = ((astype148[((((((convert_int(get_group_id(1))) * 8192) + (ax0 * 2048)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] + (NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_attention_dense_bias4[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)])) + input_embeds[((((((convert_int(get_group_id(1))) * 8192) + (ax0 * 2048)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]); + } + } + } +} + +// Function: gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2 +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len) { + float NT_matmul_intermediate_reindex_pad_local[16]; + for (int var = 0; var < 1; ++var) { + vstore8(((float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)), 0, NT_matmul_intermediate_reindex_pad_local + 0); + vstore8(((float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)), 0, NT_matmul_intermediate_reindex_pad_local + 8); + for (int ax3_0 = 0; ax3_0 < 128; ++ax3_0) { + for (int ax3_1 = 0; ax3_1 < 16; ++ax3_1) { + half condval; + if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < seq_len)) { + condval = layer_norm99[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; + } else { + condval = (half)0.000000e+00f; + } + int8 v_ = (int8)(((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*0), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*1), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*2), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*3), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*4), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*5), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*6), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*7)); + vstore8((((convert_float8(((half8)(condval, condval, condval, condval, condval, condval, condval, condval)))) * (convert_float8(((half8)(gpt_neox_layers_0_mlp_dense_h_to_4h_weight3[v_.s0],gpt_neox_layers_0_mlp_dense_h_to_4h_weight3[v_.s1],gpt_neox_layers_0_mlp_dense_h_to_4h_weight3[v_.s2],gpt_neox_layers_0_mlp_dense_h_to_4h_weight3[v_.s3],gpt_neox_layers_0_mlp_dense_h_to_4h_weight3[v_.s4],gpt_neox_layers_0_mlp_dense_h_to_4h_weight3[v_.s5],gpt_neox_layers_0_mlp_dense_h_to_4h_weight3[v_.s6],gpt_neox_layers_0_mlp_dense_h_to_4h_weight3[v_.s7]))))) + vload8(0, NT_matmul_intermediate_reindex_pad_local + 0)), 0, NT_matmul_intermediate_reindex_pad_local + 0); + half condval_1; + if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < seq_len)) { + condval_1 = layer_norm99[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; + } else { + condval_1 = (half)0.000000e+00f; + } + int8 v__1 = (int8)(((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*0), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*1), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*2), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*3), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*4), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*5), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*6), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*7)); + vstore8((((convert_float8(((half8)(condval_1, condval_1, condval_1, condval_1, condval_1, condval_1, condval_1, condval_1)))) * (convert_float8(((half8)(gpt_neox_layers_0_mlp_dense_h_to_4h_weight3[v__1.s0],gpt_neox_layers_0_mlp_dense_h_to_4h_weight3[v__1.s1],gpt_neox_layers_0_mlp_dense_h_to_4h_weight3[v__1.s2],gpt_neox_layers_0_mlp_dense_h_to_4h_weight3[v__1.s3],gpt_neox_layers_0_mlp_dense_h_to_4h_weight3[v__1.s4],gpt_neox_layers_0_mlp_dense_h_to_4h_weight3[v__1.s5],gpt_neox_layers_0_mlp_dense_h_to_4h_weight3[v__1.s6],gpt_neox_layers_0_mlp_dense_h_to_4h_weight3[v__1.s7]))))) + vload8(0, NT_matmul_intermediate_reindex_pad_local + 8)), 0, NT_matmul_intermediate_reindex_pad_local + 8); + } + } + if ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < seq_len) { + compute[(((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8))] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[0] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[(((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8))]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[0] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[(((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8))]) * 7.071068e-01f)) * 5.000000e-01f))))); + compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 1)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[1] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 1)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[1] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 1)]) * 7.071068e-01f)) * 5.000000e-01f))))); + compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 2)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[2] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 2)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[2] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 2)]) * 7.071068e-01f)) * 5.000000e-01f))))); + compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 3)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[3] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 3)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[3] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 3)]) * 7.071068e-01f)) * 5.000000e-01f))))); + compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 4)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[4] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 4)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[4] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 4)]) * 7.071068e-01f)) * 5.000000e-01f))))); + compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 5)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[5] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 5)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[5] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 5)]) * 7.071068e-01f)) * 5.000000e-01f))))); + compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 6)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[6] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 6)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[6] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 6)]) * 7.071068e-01f)) * 5.000000e-01f))))); + compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 7)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[7] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 7)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[7] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 7)]) * 7.071068e-01f)) * 5.000000e-01f))))); + } + if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < seq_len) { + compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 8192)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[8] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[(((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8))]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[8] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[(((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8))]) * 7.071068e-01f)) * 5.000000e-01f))))); + compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 8193)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[9] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 1)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[9] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 1)]) * 7.071068e-01f)) * 5.000000e-01f))))); + compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 8194)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[10] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 2)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[10] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 2)]) * 7.071068e-01f)) * 5.000000e-01f))))); + compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 8195)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[11] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 3)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[11] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 3)]) * 7.071068e-01f)) * 5.000000e-01f))))); + compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 8196)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[12] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 4)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[12] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 4)]) * 7.071068e-01f)) * 5.000000e-01f))))); + compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 8197)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[13] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 5)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[13] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 5)]) * 7.071068e-01f)) * 5.000000e-01f))))); + compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 8198)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[14] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 6)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[14] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 6)]) * 7.071068e-01f)) * 5.000000e-01f))))); + compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 8199)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[15] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 7)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[15] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 7)]) * 7.071068e-01f)) * 5.000000e-01f))))); + } + } +} + +// Function: gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen) { + int position = position_map[(((convert_int(get_group_id(0))) >> 3) + position_map_elem_offset)]; + k_data[((((layer_id * seqlen) * 2048) + ((convert_int(get_group_id(0))) * 256)) + (convert_int(get_local_id(0))))] = pages[((((((((((position / page_size) + ((position % page_size) >> 31)) * 32) + (((convert_int(get_group_id(0))) & 7) * 2)) + ((convert_int(get_local_id(0))) >> 7)) * page_size) * 128) + (((position % page_size) + (page_size & ((position % page_size) >> 31))) * 128)) + pages_elem_offset) + ((convert_int(get_local_id(0))) & 127))]; + v_data[((((layer_id * seqlen) * 2048) + ((convert_int(get_group_id(0))) * 256)) + (convert_int(get_local_id(0))))] = pages[(((((((((((position / page_size) + ((position % page_size) >> 31)) * 32) + (((convert_int(get_group_id(0))) & 7) * 2)) + ((convert_int(get_local_id(0))) >> 7)) + 16) * page_size) * 128) + (((position % page_size) + (page_size & ((position % page_size) >> 31))) * 128)) + pages_elem_offset) + ((convert_int(get_local_id(0))) & 127))]; +} + +// Function: gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size) { + half NT_matmul_intermediate_reindex_pad_local[16]; + for (int var = 0; var < 1; ++var) { + vstore8(((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, NT_matmul_intermediate_reindex_pad_local + 0); + vstore8(((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, NT_matmul_intermediate_reindex_pad_local + 8); + for (int ax3_0 = 0; ax3_0 < 128; ++ax3_0) { + for (int ax3_1 = 0; ax3_1 < 16; ++ax3_1) { + half condval; + if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { + condval = take1[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; + } else { + condval = (half)0.000000e+00f; + } + half condval_1; + if (((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) < vocab_size)) { + condval_1 = embed_out_weight3[(((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1)]; + } else { + condval_1 = (half)0.000000e+00f; + } + NT_matmul_intermediate_reindex_pad_local[0] = (NT_matmul_intermediate_reindex_pad_local[0] + (condval * condval_1)); + half condval_2; + if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { + condval_2 = take1[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; + } else { + condval_2 = (half)0.000000e+00f; + } + half condval_3; + if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 1) < vocab_size)) { + condval_3 = embed_out_weight3[((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1) + 2048)]; + } else { + condval_3 = (half)0.000000e+00f; + } + NT_matmul_intermediate_reindex_pad_local[1] = (NT_matmul_intermediate_reindex_pad_local[1] + (condval_2 * condval_3)); + half condval_4; + if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { + condval_4 = take1[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; + } else { + condval_4 = (half)0.000000e+00f; + } + half condval_5; + if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 2) < vocab_size)) { + condval_5 = embed_out_weight3[((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1) + 4096)]; + } else { + condval_5 = (half)0.000000e+00f; + } + NT_matmul_intermediate_reindex_pad_local[2] = (NT_matmul_intermediate_reindex_pad_local[2] + (condval_4 * condval_5)); + half condval_6; + if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { + condval_6 = take1[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; + } else { + condval_6 = (half)0.000000e+00f; + } + half condval_7; + if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 3) < vocab_size)) { + condval_7 = embed_out_weight3[((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1) + 6144)]; + } else { + condval_7 = (half)0.000000e+00f; + } + NT_matmul_intermediate_reindex_pad_local[3] = (NT_matmul_intermediate_reindex_pad_local[3] + (condval_6 * condval_7)); + half condval_8; + if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { + condval_8 = take1[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; + } else { + condval_8 = (half)0.000000e+00f; + } + half condval_9; + if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 4) < vocab_size)) { + condval_9 = embed_out_weight3[((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1) + 8192)]; + } else { + condval_9 = (half)0.000000e+00f; + } + NT_matmul_intermediate_reindex_pad_local[4] = (NT_matmul_intermediate_reindex_pad_local[4] + (condval_8 * condval_9)); + half condval_10; + if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { + condval_10 = take1[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; + } else { + condval_10 = (half)0.000000e+00f; + } + half condval_11; + if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 5) < vocab_size)) { + condval_11 = embed_out_weight3[((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1) + 10240)]; + } else { + condval_11 = (half)0.000000e+00f; + } + NT_matmul_intermediate_reindex_pad_local[5] = (NT_matmul_intermediate_reindex_pad_local[5] + (condval_10 * condval_11)); + half condval_12; + if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { + condval_12 = take1[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; + } else { + condval_12 = (half)0.000000e+00f; + } + half condval_13; + if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 6) < vocab_size)) { + condval_13 = embed_out_weight3[((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1) + 12288)]; + } else { + condval_13 = (half)0.000000e+00f; + } + NT_matmul_intermediate_reindex_pad_local[6] = (NT_matmul_intermediate_reindex_pad_local[6] + (condval_12 * condval_13)); + half condval_14; + if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { + condval_14 = take1[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; + } else { + condval_14 = (half)0.000000e+00f; + } + half condval_15; + if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 7) < vocab_size)) { + condval_15 = embed_out_weight3[((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1) + 14336)]; + } else { + condval_15 = (half)0.000000e+00f; + } + NT_matmul_intermediate_reindex_pad_local[7] = (NT_matmul_intermediate_reindex_pad_local[7] + (condval_14 * condval_15)); + half condval_16; + if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { + condval_16 = take1[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; + } else { + condval_16 = (half)0.000000e+00f; + } + half condval_17; + if (((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) < vocab_size)) { + condval_17 = embed_out_weight3[(((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1)]; + } else { + condval_17 = (half)0.000000e+00f; + } + NT_matmul_intermediate_reindex_pad_local[8] = (NT_matmul_intermediate_reindex_pad_local[8] + (condval_16 * condval_17)); + half condval_18; + if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { + condval_18 = take1[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; + } else { + condval_18 = (half)0.000000e+00f; + } + half condval_19; + if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 1) < vocab_size)) { + condval_19 = embed_out_weight3[((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1) + 2048)]; + } else { + condval_19 = (half)0.000000e+00f; + } + NT_matmul_intermediate_reindex_pad_local[9] = (NT_matmul_intermediate_reindex_pad_local[9] + (condval_18 * condval_19)); + half condval_20; + if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { + condval_20 = take1[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; + } else { + condval_20 = (half)0.000000e+00f; + } + half condval_21; + if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 2) < vocab_size)) { + condval_21 = embed_out_weight3[((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1) + 4096)]; + } else { + condval_21 = (half)0.000000e+00f; + } + NT_matmul_intermediate_reindex_pad_local[10] = (NT_matmul_intermediate_reindex_pad_local[10] + (condval_20 * condval_21)); + half condval_22; + if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { + condval_22 = take1[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; + } else { + condval_22 = (half)0.000000e+00f; + } + half condval_23; + if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 3) < vocab_size)) { + condval_23 = embed_out_weight3[((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1) + 6144)]; + } else { + condval_23 = (half)0.000000e+00f; + } + NT_matmul_intermediate_reindex_pad_local[11] = (NT_matmul_intermediate_reindex_pad_local[11] + (condval_22 * condval_23)); + half condval_24; + if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { + condval_24 = take1[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; + } else { + condval_24 = (half)0.000000e+00f; + } + half condval_25; + if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 4) < vocab_size)) { + condval_25 = embed_out_weight3[((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1) + 8192)]; + } else { + condval_25 = (half)0.000000e+00f; + } + NT_matmul_intermediate_reindex_pad_local[12] = (NT_matmul_intermediate_reindex_pad_local[12] + (condval_24 * condval_25)); + half condval_26; + if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { + condval_26 = take1[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; + } else { + condval_26 = (half)0.000000e+00f; + } + half condval_27; + if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 5) < vocab_size)) { + condval_27 = embed_out_weight3[((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1) + 10240)]; + } else { + condval_27 = (half)0.000000e+00f; + } + NT_matmul_intermediate_reindex_pad_local[13] = (NT_matmul_intermediate_reindex_pad_local[13] + (condval_26 * condval_27)); + half condval_28; + if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { + condval_28 = take1[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; + } else { + condval_28 = (half)0.000000e+00f; + } + half condval_29; + if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 6) < vocab_size)) { + condval_29 = embed_out_weight3[((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1) + 12288)]; + } else { + condval_29 = (half)0.000000e+00f; + } + NT_matmul_intermediate_reindex_pad_local[14] = (NT_matmul_intermediate_reindex_pad_local[14] + (condval_28 * condval_29)); + half condval_30; + if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { + condval_30 = take1[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; + } else { + condval_30 = (half)0.000000e+00f; + } + half condval_31; + if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 7) < vocab_size)) { + condval_31 = embed_out_weight3[((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1) + 14336)]; + } else { + condval_31 = (half)0.000000e+00f; + } + NT_matmul_intermediate_reindex_pad_local[15] = (NT_matmul_intermediate_reindex_pad_local[15] + (condval_30 * condval_31)); + } + } + if (((bool)((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) && ((bool)((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) < vocab_size))) { + compute[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) * vocab_size))] = (convert_float(NT_matmul_intermediate_reindex_pad_local[0])); + } + if (((bool)((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 1) < vocab_size))) { + compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) * vocab_size)) + 1)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[1])); + } + if (((bool)((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 2) < vocab_size))) { + compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) * vocab_size)) + 2)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[2])); + } + if (((bool)((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 3) < vocab_size))) { + compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) * vocab_size)) + 3)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[3])); + } + if (((bool)((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 4) < vocab_size))) { + compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) * vocab_size)) + 4)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[4])); + } + if (((bool)((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 5) < vocab_size))) { + compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) * vocab_size)) + 5)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[5])); + } + if (((bool)((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 6) < vocab_size))) { + compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) * vocab_size)) + 6)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[6])); + } + if (((bool)((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 7) < vocab_size))) { + compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) * vocab_size)) + 7)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[7])); + } + if (((bool)(((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) && ((bool)((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) < vocab_size))) { + compute[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) * vocab_size))] = (convert_float(NT_matmul_intermediate_reindex_pad_local[8])); + } + if (((bool)(((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 1) < vocab_size))) { + compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) * vocab_size)) + 1)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[9])); + } + if (((bool)(((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 2) < vocab_size))) { + compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) * vocab_size)) + 2)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[10])); + } + if (((bool)(((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 3) < vocab_size))) { + compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) * vocab_size)) + 3)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[11])); + } + if (((bool)(((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 4) < vocab_size))) { + compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) * vocab_size)) + 4)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[12])); + } + if (((bool)(((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 5) < vocab_size))) { + compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) * vocab_size)) + 5)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[13])); + } + if (((bool)(((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 6) < vocab_size))) { + compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) * vocab_size)) + 6)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[14])); + } + if (((bool)(((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 7) < vocab_size))) { + compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) * vocab_size)) + 7)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[15])); + } + } +} + +// Function: gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2 +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size) { + float8 NT_matmul_intermediate_reindex_pad_local[2]; + for (int var = 0; var < 1; ++var) { + NT_matmul_intermediate_reindex_pad_local[0] = ((float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)); + NT_matmul_intermediate_reindex_pad_local[1] = ((float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)); + for (int ax3_0 = 0; ax3_0 < 512; ++ax3_0) { + for (int ax3_1 = 0; ax3_1 < 16; ++ax3_1) { + half condval; + if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { + condval = astype147[(((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + (ax3_0 * 16)) + ax3_1)]; + } else { + condval = (half)0.000000e+00f; + } + int8 v_ = (int8)(((((((convert_int(get_group_id(0))) * 2097152) + ((convert_int(get_local_id(0))) * 65536)) + (ax3_0 * 16)) + ax3_1))+(8192*0), ((((((convert_int(get_group_id(0))) * 2097152) + ((convert_int(get_local_id(0))) * 65536)) + (ax3_0 * 16)) + ax3_1))+(8192*1), ((((((convert_int(get_group_id(0))) * 2097152) + ((convert_int(get_local_id(0))) * 65536)) + (ax3_0 * 16)) + ax3_1))+(8192*2), ((((((convert_int(get_group_id(0))) * 2097152) + ((convert_int(get_local_id(0))) * 65536)) + (ax3_0 * 16)) + ax3_1))+(8192*3), ((((((convert_int(get_group_id(0))) * 2097152) + ((convert_int(get_local_id(0))) * 65536)) + (ax3_0 * 16)) + ax3_1))+(8192*4), ((((((convert_int(get_group_id(0))) * 2097152) + ((convert_int(get_local_id(0))) * 65536)) + (ax3_0 * 16)) + ax3_1))+(8192*5), ((((((convert_int(get_group_id(0))) * 2097152) + ((convert_int(get_local_id(0))) * 65536)) + (ax3_0 * 16)) + ax3_1))+(8192*6), ((((((convert_int(get_group_id(0))) * 2097152) + ((convert_int(get_local_id(0))) * 65536)) + (ax3_0 * 16)) + ax3_1))+(8192*7)); + NT_matmul_intermediate_reindex_pad_local[0] = (((convert_float8(((half8)(condval, condval, condval, condval, condval, condval, condval, condval)))) * (convert_float8(((half8)(gpt_neox_layers_0_mlp_dense_4h_to_h_weight4[v_.s0],gpt_neox_layers_0_mlp_dense_4h_to_h_weight4[v_.s1],gpt_neox_layers_0_mlp_dense_4h_to_h_weight4[v_.s2],gpt_neox_layers_0_mlp_dense_4h_to_h_weight4[v_.s3],gpt_neox_layers_0_mlp_dense_4h_to_h_weight4[v_.s4],gpt_neox_layers_0_mlp_dense_4h_to_h_weight4[v_.s5],gpt_neox_layers_0_mlp_dense_4h_to_h_weight4[v_.s6],gpt_neox_layers_0_mlp_dense_4h_to_h_weight4[v_.s7]))))) + NT_matmul_intermediate_reindex_pad_local[0]); + half condval_1; + if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { + condval_1 = astype147[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + (ax3_0 * 16)) + ax3_1) + 8192)]; + } else { + condval_1 = (half)0.000000e+00f; + } + int8 v__1 = (int8)(((((((convert_int(get_group_id(0))) * 2097152) + ((convert_int(get_local_id(0))) * 65536)) + (ax3_0 * 16)) + ax3_1))+(8192*0), ((((((convert_int(get_group_id(0))) * 2097152) + ((convert_int(get_local_id(0))) * 65536)) + (ax3_0 * 16)) + ax3_1))+(8192*1), ((((((convert_int(get_group_id(0))) * 2097152) + ((convert_int(get_local_id(0))) * 65536)) + (ax3_0 * 16)) + ax3_1))+(8192*2), ((((((convert_int(get_group_id(0))) * 2097152) + ((convert_int(get_local_id(0))) * 65536)) + (ax3_0 * 16)) + ax3_1))+(8192*3), ((((((convert_int(get_group_id(0))) * 2097152) + ((convert_int(get_local_id(0))) * 65536)) + (ax3_0 * 16)) + ax3_1))+(8192*4), ((((((convert_int(get_group_id(0))) * 2097152) + ((convert_int(get_local_id(0))) * 65536)) + (ax3_0 * 16)) + ax3_1))+(8192*5), ((((((convert_int(get_group_id(0))) * 2097152) + ((convert_int(get_local_id(0))) * 65536)) + (ax3_0 * 16)) + ax3_1))+(8192*6), ((((((convert_int(get_group_id(0))) * 2097152) + ((convert_int(get_local_id(0))) * 65536)) + (ax3_0 * 16)) + ax3_1))+(8192*7)); + NT_matmul_intermediate_reindex_pad_local[1] = (((convert_float8(((half8)(condval_1, condval_1, condval_1, condval_1, condval_1, condval_1, condval_1, condval_1)))) * (convert_float8(((half8)(gpt_neox_layers_0_mlp_dense_4h_to_h_weight4[v__1.s0],gpt_neox_layers_0_mlp_dense_4h_to_h_weight4[v__1.s1],gpt_neox_layers_0_mlp_dense_4h_to_h_weight4[v__1.s2],gpt_neox_layers_0_mlp_dense_4h_to_h_weight4[v__1.s3],gpt_neox_layers_0_mlp_dense_4h_to_h_weight4[v__1.s4],gpt_neox_layers_0_mlp_dense_4h_to_h_weight4[v__1.s5],gpt_neox_layers_0_mlp_dense_4h_to_h_weight4[v__1.s6],gpt_neox_layers_0_mlp_dense_4h_to_h_weight4[v__1.s7]))))) + NT_matmul_intermediate_reindex_pad_local[1]); + } + } + if ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size) { + vstore8((convert_half8((NT_matmul_intermediate_reindex_pad_local[0] + vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_bias4 + (((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)))))), 0, compute + (((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8))); + } + if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size) { + vstore8((convert_half8((NT_matmul_intermediate_reindex_pad_local[1] + vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_bias4 + (((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)))))), 0, compute + ((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 2048)); + } + } +} + +// Function: gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2 +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len) { + float8 NT_matmul_intermediate_reindex_pad_local[2]; + for (int var = 0; var < 1; ++var) { + NT_matmul_intermediate_reindex_pad_local[0] = ((float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)); + NT_matmul_intermediate_reindex_pad_local[1] = ((float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)); + for (int ax3_0 = 0; ax3_0 < 512; ++ax3_0) { + for (int ax3_1 = 0; ax3_1 < 16; ++ax3_1) { + half condval; + if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < seq_len)) { + condval = astype98[(((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + (ax3_0 * 16)) + ax3_1)]; + } else { + condval = (half)0.000000e+00f; + } + int8 v_ = (int8)(((((((convert_int(get_group_id(0))) * 2097152) + ((convert_int(get_local_id(0))) * 65536)) + (ax3_0 * 16)) + ax3_1))+(8192*0), ((((((convert_int(get_group_id(0))) * 2097152) + ((convert_int(get_local_id(0))) * 65536)) + (ax3_0 * 16)) + ax3_1))+(8192*1), ((((((convert_int(get_group_id(0))) * 2097152) + ((convert_int(get_local_id(0))) * 65536)) + (ax3_0 * 16)) + ax3_1))+(8192*2), ((((((convert_int(get_group_id(0))) * 2097152) + ((convert_int(get_local_id(0))) * 65536)) + (ax3_0 * 16)) + ax3_1))+(8192*3), ((((((convert_int(get_group_id(0))) * 2097152) + ((convert_int(get_local_id(0))) * 65536)) + (ax3_0 * 16)) + ax3_1))+(8192*4), ((((((convert_int(get_group_id(0))) * 2097152) + ((convert_int(get_local_id(0))) * 65536)) + (ax3_0 * 16)) + ax3_1))+(8192*5), ((((((convert_int(get_group_id(0))) * 2097152) + ((convert_int(get_local_id(0))) * 65536)) + (ax3_0 * 16)) + ax3_1))+(8192*6), ((((((convert_int(get_group_id(0))) * 2097152) + ((convert_int(get_local_id(0))) * 65536)) + (ax3_0 * 16)) + ax3_1))+(8192*7)); + NT_matmul_intermediate_reindex_pad_local[0] = (((convert_float8(((half8)(condval, condval, condval, condval, condval, condval, condval, condval)))) * (convert_float8(((half8)(gpt_neox_layers_0_mlp_dense_4h_to_h_weight3[v_.s0],gpt_neox_layers_0_mlp_dense_4h_to_h_weight3[v_.s1],gpt_neox_layers_0_mlp_dense_4h_to_h_weight3[v_.s2],gpt_neox_layers_0_mlp_dense_4h_to_h_weight3[v_.s3],gpt_neox_layers_0_mlp_dense_4h_to_h_weight3[v_.s4],gpt_neox_layers_0_mlp_dense_4h_to_h_weight3[v_.s5],gpt_neox_layers_0_mlp_dense_4h_to_h_weight3[v_.s6],gpt_neox_layers_0_mlp_dense_4h_to_h_weight3[v_.s7]))))) + NT_matmul_intermediate_reindex_pad_local[0]); + half condval_1; + if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < seq_len)) { + condval_1 = astype98[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + (ax3_0 * 16)) + ax3_1) + 8192)]; + } else { + condval_1 = (half)0.000000e+00f; + } + int8 v__1 = (int8)(((((((convert_int(get_group_id(0))) * 2097152) + ((convert_int(get_local_id(0))) * 65536)) + (ax3_0 * 16)) + ax3_1))+(8192*0), ((((((convert_int(get_group_id(0))) * 2097152) + ((convert_int(get_local_id(0))) * 65536)) + (ax3_0 * 16)) + ax3_1))+(8192*1), ((((((convert_int(get_group_id(0))) * 2097152) + ((convert_int(get_local_id(0))) * 65536)) + (ax3_0 * 16)) + ax3_1))+(8192*2), ((((((convert_int(get_group_id(0))) * 2097152) + ((convert_int(get_local_id(0))) * 65536)) + (ax3_0 * 16)) + ax3_1))+(8192*3), ((((((convert_int(get_group_id(0))) * 2097152) + ((convert_int(get_local_id(0))) * 65536)) + (ax3_0 * 16)) + ax3_1))+(8192*4), ((((((convert_int(get_group_id(0))) * 2097152) + ((convert_int(get_local_id(0))) * 65536)) + (ax3_0 * 16)) + ax3_1))+(8192*5), ((((((convert_int(get_group_id(0))) * 2097152) + ((convert_int(get_local_id(0))) * 65536)) + (ax3_0 * 16)) + ax3_1))+(8192*6), ((((((convert_int(get_group_id(0))) * 2097152) + ((convert_int(get_local_id(0))) * 65536)) + (ax3_0 * 16)) + ax3_1))+(8192*7)); + NT_matmul_intermediate_reindex_pad_local[1] = (((convert_float8(((half8)(condval_1, condval_1, condval_1, condval_1, condval_1, condval_1, condval_1, condval_1)))) * (convert_float8(((half8)(gpt_neox_layers_0_mlp_dense_4h_to_h_weight3[v__1.s0],gpt_neox_layers_0_mlp_dense_4h_to_h_weight3[v__1.s1],gpt_neox_layers_0_mlp_dense_4h_to_h_weight3[v__1.s2],gpt_neox_layers_0_mlp_dense_4h_to_h_weight3[v__1.s3],gpt_neox_layers_0_mlp_dense_4h_to_h_weight3[v__1.s4],gpt_neox_layers_0_mlp_dense_4h_to_h_weight3[v__1.s5],gpt_neox_layers_0_mlp_dense_4h_to_h_weight3[v__1.s6],gpt_neox_layers_0_mlp_dense_4h_to_h_weight3[v__1.s7]))))) + NT_matmul_intermediate_reindex_pad_local[1]); + } + } + if ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < seq_len) { + vstore8((convert_half8((NT_matmul_intermediate_reindex_pad_local[0] + vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_bias3 + (((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)))))), 0, compute + (((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8))); + } + if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < seq_len) { + vstore8((convert_half8((NT_matmul_intermediate_reindex_pad_local[1] + vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_bias3 + (((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)))))), 0, compute + ((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 2048)); + } + } +} + +// Function: gpt_neox_q0f16_scatter_probs_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n) { + if ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) < (batch_size * n)) { + dst[((indices[(((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % (n * batch_size)) / n)] * n) + ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % n))] = src[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]; + } +} + +// Function: gpt_neox_q0f16_take_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len) { + T_take[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] = layer_norm146[(((logit_positions[((convert_int(get_group_id(0))) >> 3)] * 2048) + (((convert_int(get_group_id(0))) & 7) * 256)) + (convert_int(get_local_id(0))))]; +} + +// Function: gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset) { + if (position_map[(((convert_int(get_group_id(0))) >> 3) + position_map_elem_offset)] != -1) { + int position = position_map[(((convert_int(get_group_id(0))) >> 3) + position_map_elem_offset)]; + pages[(((((((position >> 4) * 65536) + (((convert_int(get_group_id(0))) & 7) * 4096)) + (((convert_int(get_local_id(0))) >> 7) * 2048)) + ((position & 15) * 128)) + pages_elem_offset) + ((convert_int(get_local_id(0))) & 127))] = k_data[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]; + int position_1 = position_map[(((convert_int(get_group_id(0))) >> 3) + position_map_elem_offset)]; + pages[((((((((position_1 >> 4) * 65536) + (((convert_int(get_group_id(0))) & 7) * 4096)) + (((convert_int(get_local_id(0))) >> 7) * 2048)) + ((position_1 & 15) * 128)) + pages_elem_offset) + ((convert_int(get_local_id(0))) & 127)) + 32768)] = v_data[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]; + } +} + +// Function: gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2 +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size) { + float NT_matmul_intermediate_reindex_pad_local[16]; + for (int var = 0; var < 1; ++var) { + vstore8(((float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)), 0, NT_matmul_intermediate_reindex_pad_local + 0); + vstore8(((float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)), 0, NT_matmul_intermediate_reindex_pad_local + 8); + for (int ax3_0 = 0; ax3_0 < 128; ++ax3_0) { + for (int ax3_1 = 0; ax3_1 < 16; ++ax3_1) { + half condval; + if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { + condval = layer_norm148[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; + } else { + condval = (half)0.000000e+00f; + } + int8 v_ = (int8)(((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*0), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*1), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*2), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*3), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*4), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*5), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*6), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*7)); + vstore8((((convert_float8(((half8)(condval, condval, condval, condval, condval, condval, condval, condval)))) * (convert_float8(((half8)(gpt_neox_layers_0_mlp_dense_h_to_4h_weight4[v_.s0],gpt_neox_layers_0_mlp_dense_h_to_4h_weight4[v_.s1],gpt_neox_layers_0_mlp_dense_h_to_4h_weight4[v_.s2],gpt_neox_layers_0_mlp_dense_h_to_4h_weight4[v_.s3],gpt_neox_layers_0_mlp_dense_h_to_4h_weight4[v_.s4],gpt_neox_layers_0_mlp_dense_h_to_4h_weight4[v_.s5],gpt_neox_layers_0_mlp_dense_h_to_4h_weight4[v_.s6],gpt_neox_layers_0_mlp_dense_h_to_4h_weight4[v_.s7]))))) + vload8(0, NT_matmul_intermediate_reindex_pad_local + 0)), 0, NT_matmul_intermediate_reindex_pad_local + 0); + half condval_1; + if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { + condval_1 = layer_norm148[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; + } else { + condval_1 = (half)0.000000e+00f; + } + int8 v__1 = (int8)(((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*0), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*1), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*2), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*3), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*4), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*5), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*6), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*7)); + vstore8((((convert_float8(((half8)(condval_1, condval_1, condval_1, condval_1, condval_1, condval_1, condval_1, condval_1)))) * (convert_float8(((half8)(gpt_neox_layers_0_mlp_dense_h_to_4h_weight4[v__1.s0],gpt_neox_layers_0_mlp_dense_h_to_4h_weight4[v__1.s1],gpt_neox_layers_0_mlp_dense_h_to_4h_weight4[v__1.s2],gpt_neox_layers_0_mlp_dense_h_to_4h_weight4[v__1.s3],gpt_neox_layers_0_mlp_dense_h_to_4h_weight4[v__1.s4],gpt_neox_layers_0_mlp_dense_h_to_4h_weight4[v__1.s5],gpt_neox_layers_0_mlp_dense_h_to_4h_weight4[v__1.s6],gpt_neox_layers_0_mlp_dense_h_to_4h_weight4[v__1.s7]))))) + vload8(0, NT_matmul_intermediate_reindex_pad_local + 8)), 0, NT_matmul_intermediate_reindex_pad_local + 8); + } + } + if ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size) { + compute[(((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8))] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[0] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[(((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8))]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[0] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[(((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8))]) * 7.071068e-01f)) * 5.000000e-01f))))); + compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 1)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[1] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 1)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[1] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 1)]) * 7.071068e-01f)) * 5.000000e-01f))))); + compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 2)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[2] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 2)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[2] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 2)]) * 7.071068e-01f)) * 5.000000e-01f))))); + compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 3)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[3] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 3)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[3] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 3)]) * 7.071068e-01f)) * 5.000000e-01f))))); + compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 4)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[4] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 4)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[4] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 4)]) * 7.071068e-01f)) * 5.000000e-01f))))); + compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 5)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[5] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 5)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[5] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 5)]) * 7.071068e-01f)) * 5.000000e-01f))))); + compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 6)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[6] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 6)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[6] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 6)]) * 7.071068e-01f)) * 5.000000e-01f))))); + compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 7)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[7] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 7)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[7] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 7)]) * 7.071068e-01f)) * 5.000000e-01f))))); + } + if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size) { + compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 8192)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[8] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[(((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8))]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[8] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[(((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8))]) * 7.071068e-01f)) * 5.000000e-01f))))); + compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 8193)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[9] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 1)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[9] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 1)]) * 7.071068e-01f)) * 5.000000e-01f))))); + compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 8194)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[10] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 2)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[10] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 2)]) * 7.071068e-01f)) * 5.000000e-01f))))); + compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 8195)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[11] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 3)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[11] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 3)]) * 7.071068e-01f)) * 5.000000e-01f))))); + compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 8196)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[12] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 4)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[12] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 4)]) * 7.071068e-01f)) * 5.000000e-01f))))); + compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 8197)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[13] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 5)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[13] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 5)]) * 7.071068e-01f)) * 5.000000e-01f))))); + compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 8198)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[14] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 6)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[14] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 6)]) * 7.071068e-01f)) * 5.000000e-01f))))); + compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 8199)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[15] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 7)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[15] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 7)]) * 7.071068e-01f)) * 5.000000e-01f))))); + } + } +} + +// Function: gpt_neox_q0f16_apply_logit_bias_inplace_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size) { + if ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) < num_token) { + logits[((pos2seq_id[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] * vocab_size) + token_ids[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))])] = (logits[((pos2seq_id[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] * vocab_size) + token_ids[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))])] + logit_bias[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]); + } +} + +// Function: gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1 +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len) { + half NT_matmul_intermediate_pad_rf_local[64]; + half NT_matmul_intermediate_pad_rf_local_1[8]; + half NT_matmul_intermediate_pad_local[8]; + __local half red_buf0[64]; + for (int ax0_1_init = 0; ax0_1_init < 4; ++ax0_1_init) { + for (int ax1_fused_2_init = 0; ax1_fused_2_init < 2; ++ax1_fused_2_init) { + int8 v_ = (int8)((((ax0_1_init * 2) + ax1_fused_2_init))+(8*0), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*1), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*2), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*3), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*4), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*5), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*6), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*7)); + half8 v__1 = (half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f); + NT_matmul_intermediate_pad_rf_local[v_.s0] = v__1.s0; + NT_matmul_intermediate_pad_rf_local[v_.s1] = v__1.s1; + NT_matmul_intermediate_pad_rf_local[v_.s2] = v__1.s2; + NT_matmul_intermediate_pad_rf_local[v_.s3] = v__1.s3; + NT_matmul_intermediate_pad_rf_local[v_.s4] = v__1.s4; + NT_matmul_intermediate_pad_rf_local[v_.s5] = v__1.s5; + NT_matmul_intermediate_pad_rf_local[v_.s6] = v__1.s6; + NT_matmul_intermediate_pad_rf_local[v_.s7] = v__1.s7; + } + } + for (int ax2_fused_0 = 0; ax2_fused_0 < 8; ++ax2_fused_0) { + if (((convert_int(get_group_id(1))) * 4) < seq_len) { + int8 v__2 = (int8)((0)+(8*0), (0)+(8*1), (0)+(8*2), (0)+(8*3), (0)+(8*4), (0)+(8*5), (0)+(8*6), (0)+(8*7)); + half8 v__3 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__2.s0],NT_matmul_intermediate_pad_rf_local[v__2.s1],NT_matmul_intermediate_pad_rf_local[v__2.s2],NT_matmul_intermediate_pad_rf_local[v__2.s3],NT_matmul_intermediate_pad_rf_local[v__2.s4],NT_matmul_intermediate_pad_rf_local[v__2.s5],NT_matmul_intermediate_pad_rf_local[v__2.s6],NT_matmul_intermediate_pad_rf_local[v__2.s7])) + (vload8(0, layer_norm98 + ((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, gpt_neox_layers_0_attention_query_key_value_weight3 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))); + NT_matmul_intermediate_pad_rf_local[v__2.s0] = v__3.s0; + NT_matmul_intermediate_pad_rf_local[v__2.s1] = v__3.s1; + NT_matmul_intermediate_pad_rf_local[v__2.s2] = v__3.s2; + NT_matmul_intermediate_pad_rf_local[v__2.s3] = v__3.s3; + NT_matmul_intermediate_pad_rf_local[v__2.s4] = v__3.s4; + NT_matmul_intermediate_pad_rf_local[v__2.s5] = v__3.s5; + NT_matmul_intermediate_pad_rf_local[v__2.s6] = v__3.s6; + NT_matmul_intermediate_pad_rf_local[v__2.s7] = v__3.s7; + int8 v__4 = (int8)((1)+(8*0), (1)+(8*1), (1)+(8*2), (1)+(8*3), (1)+(8*4), (1)+(8*5), (1)+(8*6), (1)+(8*7)); + half8 v__5 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__4.s0],NT_matmul_intermediate_pad_rf_local[v__4.s1],NT_matmul_intermediate_pad_rf_local[v__4.s2],NT_matmul_intermediate_pad_rf_local[v__4.s3],NT_matmul_intermediate_pad_rf_local[v__4.s4],NT_matmul_intermediate_pad_rf_local[v__4.s5],NT_matmul_intermediate_pad_rf_local[v__4.s6],NT_matmul_intermediate_pad_rf_local[v__4.s7])) + (vload8(0, layer_norm98 + ((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, gpt_neox_layers_0_attention_query_key_value_weight3 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))); + NT_matmul_intermediate_pad_rf_local[v__4.s0] = v__5.s0; + NT_matmul_intermediate_pad_rf_local[v__4.s1] = v__5.s1; + NT_matmul_intermediate_pad_rf_local[v__4.s2] = v__5.s2; + NT_matmul_intermediate_pad_rf_local[v__4.s3] = v__5.s3; + NT_matmul_intermediate_pad_rf_local[v__4.s4] = v__5.s4; + NT_matmul_intermediate_pad_rf_local[v__4.s5] = v__5.s5; + NT_matmul_intermediate_pad_rf_local[v__4.s6] = v__5.s6; + NT_matmul_intermediate_pad_rf_local[v__4.s7] = v__5.s7; + } + if ((((convert_int(get_group_id(1))) * 4) + 1) < seq_len) { + int8 v__6 = (int8)((2)+(8*0), (2)+(8*1), (2)+(8*2), (2)+(8*3), (2)+(8*4), (2)+(8*5), (2)+(8*6), (2)+(8*7)); + half8 v__7 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__6.s0],NT_matmul_intermediate_pad_rf_local[v__6.s1],NT_matmul_intermediate_pad_rf_local[v__6.s2],NT_matmul_intermediate_pad_rf_local[v__6.s3],NT_matmul_intermediate_pad_rf_local[v__6.s4],NT_matmul_intermediate_pad_rf_local[v__6.s5],NT_matmul_intermediate_pad_rf_local[v__6.s6],NT_matmul_intermediate_pad_rf_local[v__6.s7])) + (vload8(0, layer_norm98 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)) * vload8(0, gpt_neox_layers_0_attention_query_key_value_weight3 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))); + NT_matmul_intermediate_pad_rf_local[v__6.s0] = v__7.s0; + NT_matmul_intermediate_pad_rf_local[v__6.s1] = v__7.s1; + NT_matmul_intermediate_pad_rf_local[v__6.s2] = v__7.s2; + NT_matmul_intermediate_pad_rf_local[v__6.s3] = v__7.s3; + NT_matmul_intermediate_pad_rf_local[v__6.s4] = v__7.s4; + NT_matmul_intermediate_pad_rf_local[v__6.s5] = v__7.s5; + NT_matmul_intermediate_pad_rf_local[v__6.s6] = v__7.s6; + NT_matmul_intermediate_pad_rf_local[v__6.s7] = v__7.s7; + int8 v__8 = (int8)((3)+(8*0), (3)+(8*1), (3)+(8*2), (3)+(8*3), (3)+(8*4), (3)+(8*5), (3)+(8*6), (3)+(8*7)); + half8 v__9 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__8.s0],NT_matmul_intermediate_pad_rf_local[v__8.s1],NT_matmul_intermediate_pad_rf_local[v__8.s2],NT_matmul_intermediate_pad_rf_local[v__8.s3],NT_matmul_intermediate_pad_rf_local[v__8.s4],NT_matmul_intermediate_pad_rf_local[v__8.s5],NT_matmul_intermediate_pad_rf_local[v__8.s6],NT_matmul_intermediate_pad_rf_local[v__8.s7])) + (vload8(0, layer_norm98 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)) * vload8(0, gpt_neox_layers_0_attention_query_key_value_weight3 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))); + NT_matmul_intermediate_pad_rf_local[v__8.s0] = v__9.s0; + NT_matmul_intermediate_pad_rf_local[v__8.s1] = v__9.s1; + NT_matmul_intermediate_pad_rf_local[v__8.s2] = v__9.s2; + NT_matmul_intermediate_pad_rf_local[v__8.s3] = v__9.s3; + NT_matmul_intermediate_pad_rf_local[v__8.s4] = v__9.s4; + NT_matmul_intermediate_pad_rf_local[v__8.s5] = v__9.s5; + NT_matmul_intermediate_pad_rf_local[v__8.s6] = v__9.s6; + NT_matmul_intermediate_pad_rf_local[v__8.s7] = v__9.s7; + } + if ((((convert_int(get_group_id(1))) * 4) + 2) < seq_len) { + int8 v__10 = (int8)((4)+(8*0), (4)+(8*1), (4)+(8*2), (4)+(8*3), (4)+(8*4), (4)+(8*5), (4)+(8*6), (4)+(8*7)); + half8 v__11 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__10.s0],NT_matmul_intermediate_pad_rf_local[v__10.s1],NT_matmul_intermediate_pad_rf_local[v__10.s2],NT_matmul_intermediate_pad_rf_local[v__10.s3],NT_matmul_intermediate_pad_rf_local[v__10.s4],NT_matmul_intermediate_pad_rf_local[v__10.s5],NT_matmul_intermediate_pad_rf_local[v__10.s6],NT_matmul_intermediate_pad_rf_local[v__10.s7])) + (vload8(0, layer_norm98 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 4096)) * vload8(0, gpt_neox_layers_0_attention_query_key_value_weight3 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))); + NT_matmul_intermediate_pad_rf_local[v__10.s0] = v__11.s0; + NT_matmul_intermediate_pad_rf_local[v__10.s1] = v__11.s1; + NT_matmul_intermediate_pad_rf_local[v__10.s2] = v__11.s2; + NT_matmul_intermediate_pad_rf_local[v__10.s3] = v__11.s3; + NT_matmul_intermediate_pad_rf_local[v__10.s4] = v__11.s4; + NT_matmul_intermediate_pad_rf_local[v__10.s5] = v__11.s5; + NT_matmul_intermediate_pad_rf_local[v__10.s6] = v__11.s6; + NT_matmul_intermediate_pad_rf_local[v__10.s7] = v__11.s7; + int8 v__12 = (int8)((5)+(8*0), (5)+(8*1), (5)+(8*2), (5)+(8*3), (5)+(8*4), (5)+(8*5), (5)+(8*6), (5)+(8*7)); + half8 v__13 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__12.s0],NT_matmul_intermediate_pad_rf_local[v__12.s1],NT_matmul_intermediate_pad_rf_local[v__12.s2],NT_matmul_intermediate_pad_rf_local[v__12.s3],NT_matmul_intermediate_pad_rf_local[v__12.s4],NT_matmul_intermediate_pad_rf_local[v__12.s5],NT_matmul_intermediate_pad_rf_local[v__12.s6],NT_matmul_intermediate_pad_rf_local[v__12.s7])) + (vload8(0, layer_norm98 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 4096)) * vload8(0, gpt_neox_layers_0_attention_query_key_value_weight3 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))); + NT_matmul_intermediate_pad_rf_local[v__12.s0] = v__13.s0; + NT_matmul_intermediate_pad_rf_local[v__12.s1] = v__13.s1; + NT_matmul_intermediate_pad_rf_local[v__12.s2] = v__13.s2; + NT_matmul_intermediate_pad_rf_local[v__12.s3] = v__13.s3; + NT_matmul_intermediate_pad_rf_local[v__12.s4] = v__13.s4; + NT_matmul_intermediate_pad_rf_local[v__12.s5] = v__13.s5; + NT_matmul_intermediate_pad_rf_local[v__12.s6] = v__13.s6; + NT_matmul_intermediate_pad_rf_local[v__12.s7] = v__13.s7; + } + if ((((convert_int(get_group_id(1))) * 4) + 3) < seq_len) { + int8 v__14 = (int8)((6)+(8*0), (6)+(8*1), (6)+(8*2), (6)+(8*3), (6)+(8*4), (6)+(8*5), (6)+(8*6), (6)+(8*7)); + half8 v__15 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__14.s0],NT_matmul_intermediate_pad_rf_local[v__14.s1],NT_matmul_intermediate_pad_rf_local[v__14.s2],NT_matmul_intermediate_pad_rf_local[v__14.s3],NT_matmul_intermediate_pad_rf_local[v__14.s4],NT_matmul_intermediate_pad_rf_local[v__14.s5],NT_matmul_intermediate_pad_rf_local[v__14.s6],NT_matmul_intermediate_pad_rf_local[v__14.s7])) + (vload8(0, layer_norm98 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 6144)) * vload8(0, gpt_neox_layers_0_attention_query_key_value_weight3 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))); + NT_matmul_intermediate_pad_rf_local[v__14.s0] = v__15.s0; + NT_matmul_intermediate_pad_rf_local[v__14.s1] = v__15.s1; + NT_matmul_intermediate_pad_rf_local[v__14.s2] = v__15.s2; + NT_matmul_intermediate_pad_rf_local[v__14.s3] = v__15.s3; + NT_matmul_intermediate_pad_rf_local[v__14.s4] = v__15.s4; + NT_matmul_intermediate_pad_rf_local[v__14.s5] = v__15.s5; + NT_matmul_intermediate_pad_rf_local[v__14.s6] = v__15.s6; + NT_matmul_intermediate_pad_rf_local[v__14.s7] = v__15.s7; + int8 v__16 = (int8)((7)+(8*0), (7)+(8*1), (7)+(8*2), (7)+(8*3), (7)+(8*4), (7)+(8*5), (7)+(8*6), (7)+(8*7)); + half8 v__17 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__16.s0],NT_matmul_intermediate_pad_rf_local[v__16.s1],NT_matmul_intermediate_pad_rf_local[v__16.s2],NT_matmul_intermediate_pad_rf_local[v__16.s3],NT_matmul_intermediate_pad_rf_local[v__16.s4],NT_matmul_intermediate_pad_rf_local[v__16.s5],NT_matmul_intermediate_pad_rf_local[v__16.s6],NT_matmul_intermediate_pad_rf_local[v__16.s7])) + (vload8(0, layer_norm98 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 6144)) * vload8(0, gpt_neox_layers_0_attention_query_key_value_weight3 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))); + NT_matmul_intermediate_pad_rf_local[v__16.s0] = v__17.s0; + NT_matmul_intermediate_pad_rf_local[v__16.s1] = v__17.s1; + NT_matmul_intermediate_pad_rf_local[v__16.s2] = v__17.s2; + NT_matmul_intermediate_pad_rf_local[v__16.s3] = v__17.s3; + NT_matmul_intermediate_pad_rf_local[v__16.s4] = v__17.s4; + NT_matmul_intermediate_pad_rf_local[v__16.s5] = v__17.s5; + NT_matmul_intermediate_pad_rf_local[v__16.s6] = v__17.s6; + NT_matmul_intermediate_pad_rf_local[v__16.s7] = v__17.s7; + } + } + for (int ax3_fused_2_0 = 0; ax3_fused_2_0 < 1; ++ax3_fused_2_0) { + for (int ax2 = 0; ax2 < 4; ++ax2) { + vstore2(((half2)((half)0.000000e+00f, (half)0.000000e+00f)), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + (ax2 * 2))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 8))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 16))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 24))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 32))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 40))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 48))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 56))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + } + } + for (int ax2_fused_2 = 0; ax2_fused_2 < 2; ++ax2_fused_2) { + for (int ax1 = 0; ax1 < 4; ++ax1) { + barrier(CLK_LOCAL_MEM_FENCE); + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_intermediate_pad_rf_local_1[((ax1 * 2) + ax2_fused_2)]; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 16) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 8) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 4) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 2) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + NT_matmul_intermediate_pad_local[((ax1 * 2) + ax2_fused_2)] = ((volatile __local half*)red_buf0)[((convert_int(get_local_id(0))) * 32)]; + } + } + for (int ax0 = 0; ax0 < 4; ++ax0) { + for (int ax1_fused_2 = 0; ax1_fused_2 < 2; ++ax1_fused_2) { + if (((bool)(((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)(((bool)(((convert_int(get_group_id(1))) - ((seq_len + 3) >> 2)) < 0)) || ((bool)((convert_int(get_group_id(1))) == 0)))))) && ((bool)((((convert_int(get_group_id(1))) * 4) + ax0) < seq_len))) { + T_add[((((((convert_int(get_group_id(1))) * 24576) + (ax0 * 6144)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] = (NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_attention_query_key_value_bias3[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]); + } + } + } +} + +// Function: gpt_neox_q0f16_tree_attn_paged_kv_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset) { + int tile_id[1]; + int batch_idx[1]; + int batch_rows[1]; + int batch_tiles[1]; + int kv_chunk_len[1]; + __local float m_smem[32]; + __local float d_smem[32]; + float O_local[32]; + __local half Q_smem[4096]; + __local half K_smem[4096]; + __local half V_smem[4096]; + float S_local[8]; + float m_prev[1]; + float m_new[1]; + float d_new[1]; + __local float m_prev_smem[32]; + tile_id[0] = (convert_int(get_group_id(0))); + batch_idx[0] = 0; + batch_rows[0] = (q_indptr[(q_indptr_elem_offset + 1)] - q_indptr[q_indptr_elem_offset]); + batch_tiles[0] = ((batch_rows[0] + 31) >> 5); + while (1) { + if (!(((batch_idx[0] < batch_size)))) { break; } + while (1) { + if (!((((bool)(batch_tiles[0] <= tile_id[0])) && ((bool)(batch_idx[0] < batch_size))))) { break; } + tile_id[0] = (tile_id[0] - batch_tiles[0]); + batch_idx[0] = (batch_idx[0] + 1); + if (batch_idx[0] < batch_size) { + int b_idx = batch_idx[0]; + batch_rows[0] = (q_indptr[((b_idx + q_indptr_elem_offset) + 1)] - q_indptr[(b_idx + q_indptr_elem_offset)]); + batch_tiles[0] = ((batch_rows[0] + 31) >> 5); + } + } + if ((batch_idx[0] < batch_size)) { + int b_idx_1 = batch_idx[0]; + int LH_start = (tile_id[0] * 32); + int q_indptr_val = q_indptr[(b_idx_1 + q_indptr_elem_offset)]; + int cur_page_indptr_begin = page_indptr[(b_idx_1 + page_indptr_elem_offset)]; + int cur_page_indptr_end = page_indptr[((b_idx_1 + page_indptr_elem_offset) + 1)]; + int condval; + if ((cur_page_indptr_begin != cur_page_indptr_end)) { + condval = ((((cur_page_indptr_end * 32) + length_info[(b_idx_1 + length_info_elem_offset)]) - (cur_page_indptr_begin * 32)) - 32); + } else { + condval = 0; + } + kv_chunk_len[0] = condval; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + m_smem[(convert_int(get_local_id(0)))] = -5.000000e+04f; + d_smem[(convert_int(get_local_id(0)))] = 1.000000e+00f; + } + for (int li_1 = 0; li_1 < 4; ++li_1) { + for (int lj_1 = 0; lj_1 < 8; ++lj_1) { + O_local[((li_1 * 8) + lj_1)] = 0.000000e+00f; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + for (int li_lj_fused_0 = 0; li_lj_fused_0 < 8; ++li_lj_fused_0) { + if (((((li_lj_fused_0 * 4) + q_indptr_val) + LH_start) + (convert_int(get_local_id(1)))) < q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)]) { + vstore4(vload4(0, q + ((((((li_lj_fused_0 * 8192) + ((convert_int(get_local_id(1))) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))), 0, Q_smem + (((li_lj_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); + } else { + vstore4(((half4)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, Q_smem + (((li_lj_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); + } + } + barrier(CLK_LOCAL_MEM_FENCE); + for (int iterator = 0; iterator < ((kv_chunk_len[0] + 31) >> 5); ++iterator) { + barrier(CLK_LOCAL_MEM_FENCE); + for (int lz_ly_fused_0 = 0; lz_ly_fused_0 < 8; ++lz_ly_fused_0) { + if ((((iterator * 32) + (lz_ly_fused_0 * 4)) + (convert_int(get_local_id(1)))) < kv_chunk_len[0]) { + int page_no = page_values[((iterator + cur_page_indptr_begin) + page_values_elem_offset)]; + vstore4(vload4(0, pages + (((((page_no * 131072) + ((convert_int(get_group_id(1))) * 4096)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))), 0, K_smem + (((lz_ly_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); + } else { + vstore4(((half4)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, K_smem + (((lz_ly_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); + } + } + barrier(CLK_LOCAL_MEM_FENCE); + for (int lz_ly_fused_0_1 = 0; lz_ly_fused_0_1 < 8; ++lz_ly_fused_0_1) { + if ((((iterator * 32) + (lz_ly_fused_0_1 * 4)) + (convert_int(get_local_id(1)))) < kv_chunk_len[0]) { + int page_no_1 = page_values[((iterator + cur_page_indptr_begin) + page_values_elem_offset)]; + vstore4(vload4(0, pages + ((((((page_no_1 * 131072) + ((convert_int(get_group_id(1))) * 4096)) + (lz_ly_fused_0_1 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + 65536)), 0, V_smem + (((lz_ly_fused_0_1 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); + } else { + vstore4(((half4)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, V_smem + (((lz_ly_fused_0_1 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); + } + } + barrier(CLK_LOCAL_MEM_FENCE); + for (int li_1_init = 0; li_1_init < 2; ++li_1_init) { + for (int lj_1_init = 0; lj_1_init < 4; ++lj_1_init) { + S_local[((li_1_init * 4) + lj_1_init)] = 0.000000e+00f; + } + } + for (int lk_0 = 0; lk_0 < 16; ++lk_0) { + for (int li_1_1 = 0; li_1_1 < 2; ++li_1_1) { + for (int lj_1_1 = 0; lj_1_1 < 4; ++lj_1_1) { + for (int lk_1 = 0; lk_1 < 8; ++lk_1) { + S_local[((li_1_1 * 4) + lj_1_1)] = (S_local[((li_1_1 * 4) + lj_1_1)] + ((((convert_float(Q_smem[((((((convert_int(get_local_id(1))) * 1024) + (((convert_int(get_local_id(0))) >> 3) * 256)) + (li_1_1 * 128)) + (lk_0 * 8)) + lk_1)])) * (convert_float(K_smem[((((((convert_int(get_local_id(0))) & 7) * 512) + (lj_1_1 * 128)) + (lk_0 * 8)) + lk_1)]))) * attn_score_scaling_factor) * 1.275174e-01f)); + } + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + for (int li_1_2 = 0; li_1_2 < 2; ++li_1_2) { + for (int lj_1_2 = 0; lj_1_2 < 4; ++lj_1_2) { + ((__local float*)K_smem)[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 3) * 64)) + (li_1_2 * 32)) + (((convert_int(get_local_id(0))) & 7) * 4)) + lj_1_2)] = S_local[((li_1_2 * 4) + lj_1_2)]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + m_prev[0] = m_smem[(convert_int(get_local_id(0)))]; + m_new[0] = m_smem[(convert_int(get_local_id(0)))]; + for (int j = 0; j < 32; ++j) { + int cse_var_6 = ((iterator * 32) + j); + int cse_var_5 = ((iterator * 64) + (j * 2)); + if (((bool)(cse_var_6 < kv_chunk_len[0])) && ((bool)(((bool)(cse_var_6 < ((kv_chunk_len[0] + tree_order_indptr[(b_idx_1 + tree_order_indptr_elem_offset)]) - tree_order_indptr[((b_idx_1 + tree_order_indptr_elem_offset) + 1)]))) || ((bool)(((bool)(tree_order[((((tree_order_indptr[((b_idx_1 + tree_order_indptr_elem_offset) + 1)] * 2) + cse_var_5) + tree_order_elem_offset) - (kv_chunk_len[0] * 2))] <= tree_order[(((((((convert_int(get_local_id(0))) * 2) + (LH_start * 2)) + (tree_order_indptr[((b_idx_1 + tree_order_indptr_elem_offset) + 1)] * 2)) + (q_indptr[(b_idx_1 + q_indptr_elem_offset)] * 2)) + tree_order_elem_offset) - (q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)] * 2))])) && ((bool)(tree_order[(((((((convert_int(get_local_id(0))) * 2) + (LH_start * 2)) + (tree_order_indptr[((b_idx_1 + tree_order_indptr_elem_offset) + 1)] * 2)) + (q_indptr[(b_idx_1 + q_indptr_elem_offset)] * 2)) + tree_order_elem_offset) - (q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)] * 2))] < tree_order[(((((tree_order_indptr[((b_idx_1 + tree_order_indptr_elem_offset) + 1)] * 2) + cse_var_5) + tree_order_elem_offset) + 1) - (kv_chunk_len[0] * 2))]))))))) { + m_new[0] = max((float)m_new[0], (float)((__local float*)K_smem)[((((convert_int(get_local_id(1))) * 1024) + ((convert_int(get_local_id(0))) * 32)) + j)]); + } + } + d_new[0] = (d_smem[(convert_int(get_local_id(0)))] * exp2((m_prev[0] - m_new[0]))); + } + for (int j_1 = 0; j_1 < 32; ++j_1) { + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + int cse_var_8 = ((iterator * 32) + j_1); + int cse_var_7 = ((iterator * 64) + (j_1 * 2)); + if (((bool)(cse_var_8 < kv_chunk_len[0])) && ((bool)(((bool)(cse_var_8 < ((kv_chunk_len[0] + tree_order_indptr[(b_idx_1 + tree_order_indptr_elem_offset)]) - tree_order_indptr[((b_idx_1 + tree_order_indptr_elem_offset) + 1)]))) || ((bool)(((bool)(tree_order[((((tree_order_indptr[((b_idx_1 + tree_order_indptr_elem_offset) + 1)] * 2) + cse_var_7) + tree_order_elem_offset) - (kv_chunk_len[0] * 2))] <= tree_order[(((((((convert_int(get_local_id(0))) * 2) + (LH_start * 2)) + (tree_order_indptr[((b_idx_1 + tree_order_indptr_elem_offset) + 1)] * 2)) + (q_indptr[(b_idx_1 + q_indptr_elem_offset)] * 2)) + tree_order_elem_offset) - (q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)] * 2))])) && ((bool)(tree_order[(((((((convert_int(get_local_id(0))) * 2) + (LH_start * 2)) + (tree_order_indptr[((b_idx_1 + tree_order_indptr_elem_offset) + 1)] * 2)) + (q_indptr[(b_idx_1 + q_indptr_elem_offset)] * 2)) + tree_order_elem_offset) - (q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)] * 2))] < tree_order[(((((tree_order_indptr[((b_idx_1 + tree_order_indptr_elem_offset) + 1)] * 2) + cse_var_7) + tree_order_elem_offset) + 1) - (kv_chunk_len[0] * 2))]))))))) { + ((__local float*)K_smem)[((((convert_int(get_local_id(1))) * 1024) + ((convert_int(get_local_id(0))) * 32)) + j_1)] = exp2((((__local float*)K_smem)[((((convert_int(get_local_id(1))) * 1024) + ((convert_int(get_local_id(0))) * 32)) + j_1)] - m_new[0])); + } else { + ((__local float*)K_smem)[(((convert_int(get_local_id(0))) * 32) + j_1)] = exp2((-5.000000e+04f - m_new[0])); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + for (int j_2 = 0; j_2 < 32; ++j_2) { + d_new[0] = (d_new[0] + ((__local float*)K_smem)[(((convert_int(get_local_id(0))) * 32) + j_2)]); + } + m_smem[(convert_int(get_local_id(0)))] = m_new[0]; + d_smem[(convert_int(get_local_id(0)))] = d_new[0]; + m_prev_smem[(convert_int(get_local_id(0)))] = m_prev[0]; + } + barrier(CLK_LOCAL_MEM_FENCE); + for (int li_1_init_1 = 0; li_1_init_1 < 4; ++li_1_init_1) { + for (int lj_1_init_1 = 0; lj_1_init_1 < 8; ++lj_1_init_1) { + O_local[((li_1_init_1 * 8) + lj_1_init_1)] = (O_local[((li_1_init_1 * 8) + lj_1_init_1)] * exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_init_1)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_init_1)]))); + } + } + for (int lk_0_1 = 0; lk_0_1 < 4; ++lk_0_1) { + for (int lk_1_1 = 0; lk_1_1 < 8; ++lk_1_1) { + for (int li_1_3 = 0; li_1_3 < 4; ++li_1_3) { + for (int lj_1_3 = 0; lj_1_3 < 8; ++lj_1_3) { + O_local[((li_1_3 * 8) + lj_1_3)] = (O_local[((li_1_3 * 8) + lj_1_3)] + (((__local float*)K_smem)[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (li_1_3 * 32)) + (lk_0_1 * 8)) + lk_1_1)] * (convert_float(V_smem[((((lk_0_1 * 1024) + (lk_1_1 * 128)) + (((convert_int(get_local_id(0))) & 15) * 8)) + lj_1_3)])))); + } + } + } + } + } + for (int li_1_4 = 0; li_1_4 < 4; ++li_1_4) { + for (int lj_1_4 = 0; lj_1_4 < 8; ++lj_1_4) { + int cur_L = ((((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + q_indptr[(b_idx_1 + q_indptr_elem_offset)]) + LH_start) + li_1_4); + if (cur_L < q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)]) { + output[((((cur_L * 2048) + ((convert_int(get_group_id(1))) * 128)) + (((convert_int(get_local_id(0))) & 15) * 8)) + lj_1_4)] = (convert_half((O_local[((li_1_4 * 8) + lj_1_4)] / d_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_4)]))); + } + } + } + if ((convert_int(get_local_id(1))) < 1) { + int cur_L_1 = (((((convert_int(get_local_id(1))) * 32) + q_indptr[(b_idx_1 + q_indptr_elem_offset)]) + LH_start) + (convert_int(get_local_id(0)))); + if (cur_L_1 < q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)]) { + lse[((cur_L_1 * 16) + (convert_int(get_group_id(1))))] = (m_smem[(convert_int(get_local_id(0)))] + log2(d_smem[(convert_int(get_local_id(0)))])); + } + } + tile_id[0] = (tile_id[0] + 16); + } + } +} + +// Function: gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2 +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size) { + half8 NT_matmul_intermediate_reindex_pad_local[2]; + for (int var = 0; var < 1; ++var) { + NT_matmul_intermediate_reindex_pad_local[0] = ((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)); + NT_matmul_intermediate_reindex_pad_local[1] = ((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)); + for (int ax3_0 = 0; ax3_0 < 128; ++ax3_0) { + for (int ax3_1 = 0; ax3_1 < 16; ++ax3_1) { + half condval; + if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { + condval = reshape291[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; + } else { + condval = (half)0.000000e+00f; + } + int8 v_ = (int8)(((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*0), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*1), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*2), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*3), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*4), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*5), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*6), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*7)); + NT_matmul_intermediate_reindex_pad_local[0] = (NT_matmul_intermediate_reindex_pad_local[0] + (((half8)(condval, condval, condval, condval, condval, condval, condval, condval)) * ((half8)(gpt_neox_layers_0_attention_dense_weight4[v_.s0],gpt_neox_layers_0_attention_dense_weight4[v_.s1],gpt_neox_layers_0_attention_dense_weight4[v_.s2],gpt_neox_layers_0_attention_dense_weight4[v_.s3],gpt_neox_layers_0_attention_dense_weight4[v_.s4],gpt_neox_layers_0_attention_dense_weight4[v_.s5],gpt_neox_layers_0_attention_dense_weight4[v_.s6],gpt_neox_layers_0_attention_dense_weight4[v_.s7])))); + half condval_1; + if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { + condval_1 = reshape291[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; + } else { + condval_1 = (half)0.000000e+00f; + } + int8 v__1 = (int8)(((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*0), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*1), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*2), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*3), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*4), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*5), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*6), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*7)); + NT_matmul_intermediate_reindex_pad_local[1] = (NT_matmul_intermediate_reindex_pad_local[1] + (((half8)(condval_1, condval_1, condval_1, condval_1, condval_1, condval_1, condval_1, condval_1)) * ((half8)(gpt_neox_layers_0_attention_dense_weight4[v__1.s0],gpt_neox_layers_0_attention_dense_weight4[v__1.s1],gpt_neox_layers_0_attention_dense_weight4[v__1.s2],gpt_neox_layers_0_attention_dense_weight4[v__1.s3],gpt_neox_layers_0_attention_dense_weight4[v__1.s4],gpt_neox_layers_0_attention_dense_weight4[v__1.s5],gpt_neox_layers_0_attention_dense_weight4[v__1.s6],gpt_neox_layers_0_attention_dense_weight4[v__1.s7])))); + } + } + if ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size) { + vstore8(((vload8(0, astype148 + (((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8))) + (NT_matmul_intermediate_reindex_pad_local[0] + vload8(0, gpt_neox_layers_0_attention_dense_bias4 + (((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8))))) + vload8(0, input_embeds + (((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)))), 0, T_add + (((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8))); + } + if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size) { + vstore8(((vload8(0, astype148 + ((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 2048)) + (NT_matmul_intermediate_reindex_pad_local[1] + vload8(0, gpt_neox_layers_0_attention_dense_bias4 + (((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8))))) + vload8(0, input_embeds + ((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 2048))), 0, T_add + ((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 2048)); + } + } +} + +// Function: gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size) { + half NT_matmul_intermediate_pad_rf_local[32]; + half NT_matmul_intermediate_pad_rf_local_1[4]; + half NT_matmul_intermediate_pad_local[4]; + __local half red_buf0[64]; + for (int ax0_1_init = 0; ax0_1_init < 2; ++ax0_1_init) { + for (int ax1_fused_2_init = 0; ax1_fused_2_init < 2; ++ax1_fused_2_init) { + int8 v_ = (int8)((((ax0_1_init * 2) + ax1_fused_2_init))+(4*0), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*1), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*2), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*3), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*4), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*5), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*6), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*7)); + half8 v__1 = (half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f); + NT_matmul_intermediate_pad_rf_local[v_.s0] = v__1.s0; + NT_matmul_intermediate_pad_rf_local[v_.s1] = v__1.s1; + NT_matmul_intermediate_pad_rf_local[v_.s2] = v__1.s2; + NT_matmul_intermediate_pad_rf_local[v_.s3] = v__1.s3; + NT_matmul_intermediate_pad_rf_local[v_.s4] = v__1.s4; + NT_matmul_intermediate_pad_rf_local[v_.s5] = v__1.s5; + NT_matmul_intermediate_pad_rf_local[v_.s6] = v__1.s6; + NT_matmul_intermediate_pad_rf_local[v_.s7] = v__1.s7; + } + } + for (int ax2_fused_0 = 0; ax2_fused_0 < 8; ++ax2_fused_0) { + if (((convert_int(get_group_id(1))) * 2) < batch_size) { + int8 v__2 = (int8)((0)+(4*0), (0)+(4*1), (0)+(4*2), (0)+(4*3), (0)+(4*4), (0)+(4*5), (0)+(4*6), (0)+(4*7)); + half8 v__3 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__2.s0],NT_matmul_intermediate_pad_rf_local[v__2.s1],NT_matmul_intermediate_pad_rf_local[v__2.s2],NT_matmul_intermediate_pad_rf_local[v__2.s3],NT_matmul_intermediate_pad_rf_local[v__2.s4],NT_matmul_intermediate_pad_rf_local[v__2.s5],NT_matmul_intermediate_pad_rf_local[v__2.s6],NT_matmul_intermediate_pad_rf_local[v__2.s7])) + (vload8(0, reshape291 + ((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, gpt_neox_layers_0_attention_dense_weight4 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))); + NT_matmul_intermediate_pad_rf_local[v__2.s0] = v__3.s0; + NT_matmul_intermediate_pad_rf_local[v__2.s1] = v__3.s1; + NT_matmul_intermediate_pad_rf_local[v__2.s2] = v__3.s2; + NT_matmul_intermediate_pad_rf_local[v__2.s3] = v__3.s3; + NT_matmul_intermediate_pad_rf_local[v__2.s4] = v__3.s4; + NT_matmul_intermediate_pad_rf_local[v__2.s5] = v__3.s5; + NT_matmul_intermediate_pad_rf_local[v__2.s6] = v__3.s6; + NT_matmul_intermediate_pad_rf_local[v__2.s7] = v__3.s7; + int8 v__4 = (int8)((1)+(4*0), (1)+(4*1), (1)+(4*2), (1)+(4*3), (1)+(4*4), (1)+(4*5), (1)+(4*6), (1)+(4*7)); + half8 v__5 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__4.s0],NT_matmul_intermediate_pad_rf_local[v__4.s1],NT_matmul_intermediate_pad_rf_local[v__4.s2],NT_matmul_intermediate_pad_rf_local[v__4.s3],NT_matmul_intermediate_pad_rf_local[v__4.s4],NT_matmul_intermediate_pad_rf_local[v__4.s5],NT_matmul_intermediate_pad_rf_local[v__4.s6],NT_matmul_intermediate_pad_rf_local[v__4.s7])) + (vload8(0, reshape291 + ((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, gpt_neox_layers_0_attention_dense_weight4 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))); + NT_matmul_intermediate_pad_rf_local[v__4.s0] = v__5.s0; + NT_matmul_intermediate_pad_rf_local[v__4.s1] = v__5.s1; + NT_matmul_intermediate_pad_rf_local[v__4.s2] = v__5.s2; + NT_matmul_intermediate_pad_rf_local[v__4.s3] = v__5.s3; + NT_matmul_intermediate_pad_rf_local[v__4.s4] = v__5.s4; + NT_matmul_intermediate_pad_rf_local[v__4.s5] = v__5.s5; + NT_matmul_intermediate_pad_rf_local[v__4.s6] = v__5.s6; + NT_matmul_intermediate_pad_rf_local[v__4.s7] = v__5.s7; + } + if ((((convert_int(get_group_id(1))) * 2) + 1) < batch_size) { + int8 v__6 = (int8)((2)+(4*0), (2)+(4*1), (2)+(4*2), (2)+(4*3), (2)+(4*4), (2)+(4*5), (2)+(4*6), (2)+(4*7)); + half8 v__7 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__6.s0],NT_matmul_intermediate_pad_rf_local[v__6.s1],NT_matmul_intermediate_pad_rf_local[v__6.s2],NT_matmul_intermediate_pad_rf_local[v__6.s3],NT_matmul_intermediate_pad_rf_local[v__6.s4],NT_matmul_intermediate_pad_rf_local[v__6.s5],NT_matmul_intermediate_pad_rf_local[v__6.s6],NT_matmul_intermediate_pad_rf_local[v__6.s7])) + (vload8(0, reshape291 + (((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)) * vload8(0, gpt_neox_layers_0_attention_dense_weight4 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))); + NT_matmul_intermediate_pad_rf_local[v__6.s0] = v__7.s0; + NT_matmul_intermediate_pad_rf_local[v__6.s1] = v__7.s1; + NT_matmul_intermediate_pad_rf_local[v__6.s2] = v__7.s2; + NT_matmul_intermediate_pad_rf_local[v__6.s3] = v__7.s3; + NT_matmul_intermediate_pad_rf_local[v__6.s4] = v__7.s4; + NT_matmul_intermediate_pad_rf_local[v__6.s5] = v__7.s5; + NT_matmul_intermediate_pad_rf_local[v__6.s6] = v__7.s6; + NT_matmul_intermediate_pad_rf_local[v__6.s7] = v__7.s7; + int8 v__8 = (int8)((3)+(4*0), (3)+(4*1), (3)+(4*2), (3)+(4*3), (3)+(4*4), (3)+(4*5), (3)+(4*6), (3)+(4*7)); + half8 v__9 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__8.s0],NT_matmul_intermediate_pad_rf_local[v__8.s1],NT_matmul_intermediate_pad_rf_local[v__8.s2],NT_matmul_intermediate_pad_rf_local[v__8.s3],NT_matmul_intermediate_pad_rf_local[v__8.s4],NT_matmul_intermediate_pad_rf_local[v__8.s5],NT_matmul_intermediate_pad_rf_local[v__8.s6],NT_matmul_intermediate_pad_rf_local[v__8.s7])) + (vload8(0, reshape291 + (((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)) * vload8(0, gpt_neox_layers_0_attention_dense_weight4 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))); + NT_matmul_intermediate_pad_rf_local[v__8.s0] = v__9.s0; + NT_matmul_intermediate_pad_rf_local[v__8.s1] = v__9.s1; + NT_matmul_intermediate_pad_rf_local[v__8.s2] = v__9.s2; + NT_matmul_intermediate_pad_rf_local[v__8.s3] = v__9.s3; + NT_matmul_intermediate_pad_rf_local[v__8.s4] = v__9.s4; + NT_matmul_intermediate_pad_rf_local[v__8.s5] = v__9.s5; + NT_matmul_intermediate_pad_rf_local[v__8.s6] = v__9.s6; + NT_matmul_intermediate_pad_rf_local[v__8.s7] = v__9.s7; + } + } + for (int ax3_fused_2_0 = 0; ax3_fused_2_0 < 1; ++ax3_fused_2_0) { + for (int ax2 = 0; ax2 < 2; ++ax2) { + vstore2(((half2)((half)0.000000e+00f, (half)0.000000e+00f)), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + (ax2 * 2))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 4))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 8))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 12))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 16))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 20))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 24))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 28))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + } + } + for (int ax2_fused_2 = 0; ax2_fused_2 < 2; ++ax2_fused_2) { + for (int ax1 = 0; ax1 < 2; ++ax1) { + barrier(CLK_LOCAL_MEM_FENCE); + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_intermediate_pad_rf_local_1[((ax1 * 2) + ax2_fused_2)]; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 16) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 8) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 4) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 2) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + NT_matmul_intermediate_pad_local[((ax1 * 2) + ax2_fused_2)] = ((volatile __local half*)red_buf0)[((convert_int(get_local_id(0))) * 32)]; + } + } + for (int ax0 = 0; ax0 < 2; ++ax0) { + for (int ax1_fused_2 = 0; ax1_fused_2 < 2; ++ax1_fused_2) { + if (((bool)(((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)(((bool)(((convert_int(get_group_id(1))) - ((batch_size + 1) >> 1)) < 0)) || ((bool)((convert_int(get_group_id(1))) == 0)))))) && ((bool)((((convert_int(get_group_id(1))) * 2) + ax0) < batch_size))) { + T_add[((((((convert_int(get_group_id(1))) * 4096) + (ax0 * 2048)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] = ((astype148[((((((convert_int(get_group_id(1))) * 4096) + (ax0 * 2048)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] + (NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_attention_dense_bias4[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)])) + input_embeds[((((((convert_int(get_group_id(1))) * 4096) + (ax0 * 2048)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]); + } + } + } +} + +// Function: gpt_neox_q0f16_fused_NT_matmul10_add10_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49) { + half NT_matmul_rf_local[8]; + half gpt_neox_layers_0_attention_query_key_value_weight2_local[8]; + half NT_matmul_rf_local_1[1]; + __local half red_buf0[64]; + vstore8(((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, NT_matmul_rf_local + 0); + for (int ax1_fused_u_fused_0 = 0; ax1_fused_u_fused_0 < 8; ++ax1_fused_u_fused_0) { + gpt_neox_layers_0_attention_query_key_value_weight2_local[0] = gpt_neox_layers_0_attention_query_key_value_weight2[(((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))]; + gpt_neox_layers_0_attention_query_key_value_weight2_local[1] = gpt_neox_layers_0_attention_query_key_value_weight2[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 1)]; + gpt_neox_layers_0_attention_query_key_value_weight2_local[2] = gpt_neox_layers_0_attention_query_key_value_weight2[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2)]; + gpt_neox_layers_0_attention_query_key_value_weight2_local[3] = gpt_neox_layers_0_attention_query_key_value_weight2[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 3)]; + gpt_neox_layers_0_attention_query_key_value_weight2_local[4] = gpt_neox_layers_0_attention_query_key_value_weight2[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 4)]; + gpt_neox_layers_0_attention_query_key_value_weight2_local[5] = gpt_neox_layers_0_attention_query_key_value_weight2[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 5)]; + gpt_neox_layers_0_attention_query_key_value_weight2_local[6] = gpt_neox_layers_0_attention_query_key_value_weight2[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 6)]; + gpt_neox_layers_0_attention_query_key_value_weight2_local[7] = gpt_neox_layers_0_attention_query_key_value_weight2[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 7)]; + vstore8((vload8(0, NT_matmul_rf_local + 0) + (vload8(0, layer_norm49 + ((ax1_fused_u_fused_0 * 256) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, gpt_neox_layers_0_attention_query_key_value_weight2_local + 0))), 0, NT_matmul_rf_local + 0); + } + for (int ax2_fused_2_0 = 0; ax2_fused_2_0 < 1; ++ax2_fused_2_0) { + NT_matmul_rf_local_1[0] = (half)0.000000e+00f; + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[0]); + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[1]); + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[2]); + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[3]); + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[4]); + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[5]); + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[6]); + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[7]); + } + barrier(CLK_LOCAL_MEM_FENCE); + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_rf_local_1[0]; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 16) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 8) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 4) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 2) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) == 0) { + T_add[(((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0))))] = (((volatile __local half*)red_buf0)[((convert_int(get_local_id(0))) * 32)] + gpt_neox_layers_0_attention_query_key_value_bias2[(((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0))))]); + } +} + +// Function: gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2) { + float NT_matmul_rf_local[8]; + half gpt_neox_layers_0_mlp_dense_4h_to_h_weight2_local[8]; + float NT_matmul_rf_local_1[1]; + __local float red_buf0[64]; + vstore8(((float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)), 0, NT_matmul_rf_local + 0); + for (int ax1_fused_u_fused_0 = 0; ax1_fused_u_fused_0 < 32; ++ax1_fused_u_fused_0) { + gpt_neox_layers_0_mlp_dense_4h_to_h_weight2_local[0] = gpt_neox_layers_0_mlp_dense_4h_to_h_weight2[(((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 8192)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))]; + gpt_neox_layers_0_mlp_dense_4h_to_h_weight2_local[1] = gpt_neox_layers_0_mlp_dense_4h_to_h_weight2[((((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 8192)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 1)]; + gpt_neox_layers_0_mlp_dense_4h_to_h_weight2_local[2] = gpt_neox_layers_0_mlp_dense_4h_to_h_weight2[((((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 8192)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2)]; + gpt_neox_layers_0_mlp_dense_4h_to_h_weight2_local[3] = gpt_neox_layers_0_mlp_dense_4h_to_h_weight2[((((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 8192)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 3)]; + gpt_neox_layers_0_mlp_dense_4h_to_h_weight2_local[4] = gpt_neox_layers_0_mlp_dense_4h_to_h_weight2[((((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 8192)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 4)]; + gpt_neox_layers_0_mlp_dense_4h_to_h_weight2_local[5] = gpt_neox_layers_0_mlp_dense_4h_to_h_weight2[((((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 8192)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 5)]; + gpt_neox_layers_0_mlp_dense_4h_to_h_weight2_local[6] = gpt_neox_layers_0_mlp_dense_4h_to_h_weight2[((((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 8192)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 6)]; + gpt_neox_layers_0_mlp_dense_4h_to_h_weight2_local[7] = gpt_neox_layers_0_mlp_dense_4h_to_h_weight2[((((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 8192)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 7)]; + vstore8((vload8(0, NT_matmul_rf_local + 0) + ((convert_float8(vload8(0, astype49 + ((ax1_fused_u_fused_0 * 256) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_weight2_local + 0))))), 0, NT_matmul_rf_local + 0); + } + for (int ax2_fused_2_0 = 0; ax2_fused_2_0 < 1; ++ax2_fused_2_0) { + NT_matmul_rf_local_1[0] = 0.000000e+00f; + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[0]); + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[1]); + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[2]); + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[3]); + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[4]); + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[5]); + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[6]); + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[7]); + } + barrier(CLK_LOCAL_MEM_FENCE); + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_rf_local_1[0]; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 16) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 8) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 4) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 2) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) == 0) { + compute[(((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0))))] = (convert_half((((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) * 32)] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias2[(((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0))))]))); + } +} + +// Function: gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size) { + half NT_matmul_rf_local[8]; + half embed_out_weight2_local[8]; + half NT_matmul_rf_local_1[1]; + half in_thread_NT_matmul_intermediate_local[1]; + __local half red_buf0[64]; + if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { + vstore8(((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, NT_matmul_rf_local + 0); + } + for (int ax1_fused_u_fused_0 = 0; ax1_fused_u_fused_0 < 8; ++ax1_fused_u_fused_0) { + if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { + embed_out_weight2_local[0] = embed_out_weight2[(((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))]; + } + if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { + embed_out_weight2_local[1] = embed_out_weight2[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 1)]; + } + if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { + embed_out_weight2_local[2] = embed_out_weight2[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2)]; + } + if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { + embed_out_weight2_local[3] = embed_out_weight2[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 3)]; + } + if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { + embed_out_weight2_local[4] = embed_out_weight2[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 4)]; + } + if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { + embed_out_weight2_local[5] = embed_out_weight2[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 5)]; + } + if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { + embed_out_weight2_local[6] = embed_out_weight2[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 6)]; + } + if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { + embed_out_weight2_local[7] = embed_out_weight2[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 7)]; + } + if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { + vstore8((vload8(0, NT_matmul_rf_local + 0) + (vload8(0, layer_norm97 + ((ax1_fused_u_fused_0 * 256) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, embed_out_weight2_local + 0))), 0, NT_matmul_rf_local + 0); + } + } + for (int ax2_fused_2_0 = 0; ax2_fused_2_0 < 1; ++ax2_fused_2_0) { + if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { + NT_matmul_rf_local_1[0] = (half)0.000000e+00f; + } + if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[0]); + } + if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[1]); + } + if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[2]); + } + if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[3]); + } + if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[4]); + } + if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[5]); + } + if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[6]); + } + if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[7]); + } + } + in_thread_NT_matmul_intermediate_local[0] = (half)0.000000e+00f; + if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { + in_thread_NT_matmul_intermediate_local[0] = (in_thread_NT_matmul_intermediate_local[0] + NT_matmul_rf_local_1[0]); + } + barrier(CLK_LOCAL_MEM_FENCE); + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = in_thread_NT_matmul_intermediate_local[0]; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 16) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 8) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 4) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 2) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if (((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size))) { + compute[(((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0))))] = (convert_float(((volatile __local half*)red_buf0)[((convert_int(get_local_id(0))) * 32)])); + } +} + +// Function: gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1 +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size) { + float NT_matmul_intermediate_pad_rf_local[64]; + float NT_matmul_intermediate_pad_rf_local_1[8]; + float NT_matmul_intermediate_pad_local[8]; + __local float red_buf0[64]; + for (int ax0_1_init = 0; ax0_1_init < 4; ++ax0_1_init) { + for (int ax1_fused_2_init = 0; ax1_fused_2_init < 2; ++ax1_fused_2_init) { + int8 v_ = (int8)((((ax0_1_init * 2) + ax1_fused_2_init))+(8*0), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*1), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*2), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*3), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*4), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*5), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*6), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*7)); + float8 v__1 = (float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f); + NT_matmul_intermediate_pad_rf_local[v_.s0] = v__1.s0; + NT_matmul_intermediate_pad_rf_local[v_.s1] = v__1.s1; + NT_matmul_intermediate_pad_rf_local[v_.s2] = v__1.s2; + NT_matmul_intermediate_pad_rf_local[v_.s3] = v__1.s3; + NT_matmul_intermediate_pad_rf_local[v_.s4] = v__1.s4; + NT_matmul_intermediate_pad_rf_local[v_.s5] = v__1.s5; + NT_matmul_intermediate_pad_rf_local[v_.s6] = v__1.s6; + NT_matmul_intermediate_pad_rf_local[v_.s7] = v__1.s7; + } + } + for (int ax2_fused_0 = 0; ax2_fused_0 < 32; ++ax2_fused_0) { + if (((convert_int(get_group_id(1))) * 4) < batch_size) { + int8 v__2 = (int8)((0)+(8*0), (0)+(8*1), (0)+(8*2), (0)+(8*3), (0)+(8*4), (0)+(8*5), (0)+(8*6), (0)+(8*7)); + float8 v__3 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__2.s0],NT_matmul_intermediate_pad_rf_local[v__2.s1],NT_matmul_intermediate_pad_rf_local[v__2.s2],NT_matmul_intermediate_pad_rf_local[v__2.s3],NT_matmul_intermediate_pad_rf_local[v__2.s4],NT_matmul_intermediate_pad_rf_local[v__2.s5],NT_matmul_intermediate_pad_rf_local[v__2.s6],NT_matmul_intermediate_pad_rf_local[v__2.s7])) + ((convert_float8(vload8(0, astype147 + ((((convert_int(get_group_id(1))) * 32768) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_weight4 + (((((convert_int(get_group_id(0))) * 32768) + ((convert_int(get_local_id(0))) * 16384)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))))); + NT_matmul_intermediate_pad_rf_local[v__2.s0] = v__3.s0; + NT_matmul_intermediate_pad_rf_local[v__2.s1] = v__3.s1; + NT_matmul_intermediate_pad_rf_local[v__2.s2] = v__3.s2; + NT_matmul_intermediate_pad_rf_local[v__2.s3] = v__3.s3; + NT_matmul_intermediate_pad_rf_local[v__2.s4] = v__3.s4; + NT_matmul_intermediate_pad_rf_local[v__2.s5] = v__3.s5; + NT_matmul_intermediate_pad_rf_local[v__2.s6] = v__3.s6; + NT_matmul_intermediate_pad_rf_local[v__2.s7] = v__3.s7; + int8 v__4 = (int8)((1)+(8*0), (1)+(8*1), (1)+(8*2), (1)+(8*3), (1)+(8*4), (1)+(8*5), (1)+(8*6), (1)+(8*7)); + float8 v__5 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__4.s0],NT_matmul_intermediate_pad_rf_local[v__4.s1],NT_matmul_intermediate_pad_rf_local[v__4.s2],NT_matmul_intermediate_pad_rf_local[v__4.s3],NT_matmul_intermediate_pad_rf_local[v__4.s4],NT_matmul_intermediate_pad_rf_local[v__4.s5],NT_matmul_intermediate_pad_rf_local[v__4.s6],NT_matmul_intermediate_pad_rf_local[v__4.s7])) + ((convert_float8(vload8(0, astype147 + ((((convert_int(get_group_id(1))) * 32768) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_weight4 + ((((((convert_int(get_group_id(0))) * 32768) + ((convert_int(get_local_id(0))) * 16384)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 8192))))); + NT_matmul_intermediate_pad_rf_local[v__4.s0] = v__5.s0; + NT_matmul_intermediate_pad_rf_local[v__4.s1] = v__5.s1; + NT_matmul_intermediate_pad_rf_local[v__4.s2] = v__5.s2; + NT_matmul_intermediate_pad_rf_local[v__4.s3] = v__5.s3; + NT_matmul_intermediate_pad_rf_local[v__4.s4] = v__5.s4; + NT_matmul_intermediate_pad_rf_local[v__4.s5] = v__5.s5; + NT_matmul_intermediate_pad_rf_local[v__4.s6] = v__5.s6; + NT_matmul_intermediate_pad_rf_local[v__4.s7] = v__5.s7; + } + if ((((convert_int(get_group_id(1))) * 4) + 1) < batch_size) { + int8 v__6 = (int8)((2)+(8*0), (2)+(8*1), (2)+(8*2), (2)+(8*3), (2)+(8*4), (2)+(8*5), (2)+(8*6), (2)+(8*7)); + float8 v__7 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__6.s0],NT_matmul_intermediate_pad_rf_local[v__6.s1],NT_matmul_intermediate_pad_rf_local[v__6.s2],NT_matmul_intermediate_pad_rf_local[v__6.s3],NT_matmul_intermediate_pad_rf_local[v__6.s4],NT_matmul_intermediate_pad_rf_local[v__6.s5],NT_matmul_intermediate_pad_rf_local[v__6.s6],NT_matmul_intermediate_pad_rf_local[v__6.s7])) + ((convert_float8(vload8(0, astype147 + (((((convert_int(get_group_id(1))) * 32768) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 8192)))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_weight4 + (((((convert_int(get_group_id(0))) * 32768) + ((convert_int(get_local_id(0))) * 16384)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))))); + NT_matmul_intermediate_pad_rf_local[v__6.s0] = v__7.s0; + NT_matmul_intermediate_pad_rf_local[v__6.s1] = v__7.s1; + NT_matmul_intermediate_pad_rf_local[v__6.s2] = v__7.s2; + NT_matmul_intermediate_pad_rf_local[v__6.s3] = v__7.s3; + NT_matmul_intermediate_pad_rf_local[v__6.s4] = v__7.s4; + NT_matmul_intermediate_pad_rf_local[v__6.s5] = v__7.s5; + NT_matmul_intermediate_pad_rf_local[v__6.s6] = v__7.s6; + NT_matmul_intermediate_pad_rf_local[v__6.s7] = v__7.s7; + int8 v__8 = (int8)((3)+(8*0), (3)+(8*1), (3)+(8*2), (3)+(8*3), (3)+(8*4), (3)+(8*5), (3)+(8*6), (3)+(8*7)); + float8 v__9 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__8.s0],NT_matmul_intermediate_pad_rf_local[v__8.s1],NT_matmul_intermediate_pad_rf_local[v__8.s2],NT_matmul_intermediate_pad_rf_local[v__8.s3],NT_matmul_intermediate_pad_rf_local[v__8.s4],NT_matmul_intermediate_pad_rf_local[v__8.s5],NT_matmul_intermediate_pad_rf_local[v__8.s6],NT_matmul_intermediate_pad_rf_local[v__8.s7])) + ((convert_float8(vload8(0, astype147 + (((((convert_int(get_group_id(1))) * 32768) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 8192)))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_weight4 + ((((((convert_int(get_group_id(0))) * 32768) + ((convert_int(get_local_id(0))) * 16384)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 8192))))); + NT_matmul_intermediate_pad_rf_local[v__8.s0] = v__9.s0; + NT_matmul_intermediate_pad_rf_local[v__8.s1] = v__9.s1; + NT_matmul_intermediate_pad_rf_local[v__8.s2] = v__9.s2; + NT_matmul_intermediate_pad_rf_local[v__8.s3] = v__9.s3; + NT_matmul_intermediate_pad_rf_local[v__8.s4] = v__9.s4; + NT_matmul_intermediate_pad_rf_local[v__8.s5] = v__9.s5; + NT_matmul_intermediate_pad_rf_local[v__8.s6] = v__9.s6; + NT_matmul_intermediate_pad_rf_local[v__8.s7] = v__9.s7; + } + if ((((convert_int(get_group_id(1))) * 4) + 2) < batch_size) { + int8 v__10 = (int8)((4)+(8*0), (4)+(8*1), (4)+(8*2), (4)+(8*3), (4)+(8*4), (4)+(8*5), (4)+(8*6), (4)+(8*7)); + float8 v__11 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__10.s0],NT_matmul_intermediate_pad_rf_local[v__10.s1],NT_matmul_intermediate_pad_rf_local[v__10.s2],NT_matmul_intermediate_pad_rf_local[v__10.s3],NT_matmul_intermediate_pad_rf_local[v__10.s4],NT_matmul_intermediate_pad_rf_local[v__10.s5],NT_matmul_intermediate_pad_rf_local[v__10.s6],NT_matmul_intermediate_pad_rf_local[v__10.s7])) + ((convert_float8(vload8(0, astype147 + (((((convert_int(get_group_id(1))) * 32768) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 16384)))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_weight4 + (((((convert_int(get_group_id(0))) * 32768) + ((convert_int(get_local_id(0))) * 16384)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))))); + NT_matmul_intermediate_pad_rf_local[v__10.s0] = v__11.s0; + NT_matmul_intermediate_pad_rf_local[v__10.s1] = v__11.s1; + NT_matmul_intermediate_pad_rf_local[v__10.s2] = v__11.s2; + NT_matmul_intermediate_pad_rf_local[v__10.s3] = v__11.s3; + NT_matmul_intermediate_pad_rf_local[v__10.s4] = v__11.s4; + NT_matmul_intermediate_pad_rf_local[v__10.s5] = v__11.s5; + NT_matmul_intermediate_pad_rf_local[v__10.s6] = v__11.s6; + NT_matmul_intermediate_pad_rf_local[v__10.s7] = v__11.s7; + int8 v__12 = (int8)((5)+(8*0), (5)+(8*1), (5)+(8*2), (5)+(8*3), (5)+(8*4), (5)+(8*5), (5)+(8*6), (5)+(8*7)); + float8 v__13 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__12.s0],NT_matmul_intermediate_pad_rf_local[v__12.s1],NT_matmul_intermediate_pad_rf_local[v__12.s2],NT_matmul_intermediate_pad_rf_local[v__12.s3],NT_matmul_intermediate_pad_rf_local[v__12.s4],NT_matmul_intermediate_pad_rf_local[v__12.s5],NT_matmul_intermediate_pad_rf_local[v__12.s6],NT_matmul_intermediate_pad_rf_local[v__12.s7])) + ((convert_float8(vload8(0, astype147 + (((((convert_int(get_group_id(1))) * 32768) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 16384)))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_weight4 + ((((((convert_int(get_group_id(0))) * 32768) + ((convert_int(get_local_id(0))) * 16384)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 8192))))); + NT_matmul_intermediate_pad_rf_local[v__12.s0] = v__13.s0; + NT_matmul_intermediate_pad_rf_local[v__12.s1] = v__13.s1; + NT_matmul_intermediate_pad_rf_local[v__12.s2] = v__13.s2; + NT_matmul_intermediate_pad_rf_local[v__12.s3] = v__13.s3; + NT_matmul_intermediate_pad_rf_local[v__12.s4] = v__13.s4; + NT_matmul_intermediate_pad_rf_local[v__12.s5] = v__13.s5; + NT_matmul_intermediate_pad_rf_local[v__12.s6] = v__13.s6; + NT_matmul_intermediate_pad_rf_local[v__12.s7] = v__13.s7; + } + if ((((convert_int(get_group_id(1))) * 4) + 3) < batch_size) { + int8 v__14 = (int8)((6)+(8*0), (6)+(8*1), (6)+(8*2), (6)+(8*3), (6)+(8*4), (6)+(8*5), (6)+(8*6), (6)+(8*7)); + float8 v__15 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__14.s0],NT_matmul_intermediate_pad_rf_local[v__14.s1],NT_matmul_intermediate_pad_rf_local[v__14.s2],NT_matmul_intermediate_pad_rf_local[v__14.s3],NT_matmul_intermediate_pad_rf_local[v__14.s4],NT_matmul_intermediate_pad_rf_local[v__14.s5],NT_matmul_intermediate_pad_rf_local[v__14.s6],NT_matmul_intermediate_pad_rf_local[v__14.s7])) + ((convert_float8(vload8(0, astype147 + (((((convert_int(get_group_id(1))) * 32768) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 24576)))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_weight4 + (((((convert_int(get_group_id(0))) * 32768) + ((convert_int(get_local_id(0))) * 16384)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))))); + NT_matmul_intermediate_pad_rf_local[v__14.s0] = v__15.s0; + NT_matmul_intermediate_pad_rf_local[v__14.s1] = v__15.s1; + NT_matmul_intermediate_pad_rf_local[v__14.s2] = v__15.s2; + NT_matmul_intermediate_pad_rf_local[v__14.s3] = v__15.s3; + NT_matmul_intermediate_pad_rf_local[v__14.s4] = v__15.s4; + NT_matmul_intermediate_pad_rf_local[v__14.s5] = v__15.s5; + NT_matmul_intermediate_pad_rf_local[v__14.s6] = v__15.s6; + NT_matmul_intermediate_pad_rf_local[v__14.s7] = v__15.s7; + int8 v__16 = (int8)((7)+(8*0), (7)+(8*1), (7)+(8*2), (7)+(8*3), (7)+(8*4), (7)+(8*5), (7)+(8*6), (7)+(8*7)); + float8 v__17 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__16.s0],NT_matmul_intermediate_pad_rf_local[v__16.s1],NT_matmul_intermediate_pad_rf_local[v__16.s2],NT_matmul_intermediate_pad_rf_local[v__16.s3],NT_matmul_intermediate_pad_rf_local[v__16.s4],NT_matmul_intermediate_pad_rf_local[v__16.s5],NT_matmul_intermediate_pad_rf_local[v__16.s6],NT_matmul_intermediate_pad_rf_local[v__16.s7])) + ((convert_float8(vload8(0, astype147 + (((((convert_int(get_group_id(1))) * 32768) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 24576)))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_weight4 + ((((((convert_int(get_group_id(0))) * 32768) + ((convert_int(get_local_id(0))) * 16384)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 8192))))); + NT_matmul_intermediate_pad_rf_local[v__16.s0] = v__17.s0; + NT_matmul_intermediate_pad_rf_local[v__16.s1] = v__17.s1; + NT_matmul_intermediate_pad_rf_local[v__16.s2] = v__17.s2; + NT_matmul_intermediate_pad_rf_local[v__16.s3] = v__17.s3; + NT_matmul_intermediate_pad_rf_local[v__16.s4] = v__17.s4; + NT_matmul_intermediate_pad_rf_local[v__16.s5] = v__17.s5; + NT_matmul_intermediate_pad_rf_local[v__16.s6] = v__17.s6; + NT_matmul_intermediate_pad_rf_local[v__16.s7] = v__17.s7; + } + } + for (int ax3_fused_2_0 = 0; ax3_fused_2_0 < 1; ++ax3_fused_2_0) { + for (int ax2 = 0; ax2 < 4; ++ax2) { + vstore2(((float2)(0.000000e+00f, 0.000000e+00f)), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + (ax2 * 2))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 8))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 16))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 24))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 32))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 40))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 48))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 56))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + } + } + for (int ax2_fused_2 = 0; ax2_fused_2 < 2; ++ax2_fused_2) { + for (int ax1 = 0; ax1 < 4; ++ax1) { + barrier(CLK_LOCAL_MEM_FENCE); + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_intermediate_pad_rf_local_1[((ax1 * 2) + ax2_fused_2)]; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 16) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 8) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 4) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 2) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + NT_matmul_intermediate_pad_local[((ax1 * 2) + ax2_fused_2)] = ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) * 32)]; + } + } + for (int ax0 = 0; ax0 < 4; ++ax0) { + for (int ax1_fused_2 = 0; ax1_fused_2 < 2; ++ax1_fused_2) { + if (((bool)(((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)(((bool)(((convert_int(get_group_id(1))) - ((batch_size + 3) >> 2)) < 0)) || ((bool)((convert_int(get_group_id(1))) == 0)))))) && ((bool)((((convert_int(get_group_id(1))) * 4) + ax0) < batch_size))) { + compute[((((((convert_int(get_group_id(1))) * 8192) + (ax0 * 2048)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] = (convert_half((NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias4[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]))); + } + } + } +} + +// Function: gpt_neox_q0f16_apply_bitmask_inplace_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size) { + if ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) < (num_seq * vocab_size)) { + float condval; + if ((((bitmask[((((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % vocab_size) >> 5) + (seq_ids[(((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % (vocab_size * num_seq)) / vocab_size)] * ((vocab_size + 31) >> 5)))] >> (((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % vocab_size) & 31)) & 1) == 1)) { + condval = logits[((seq_ids[((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) / vocab_size)] * vocab_size) + ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % vocab_size))]; + } else { + condval = -3.402823e+38f; + } + logits[((seq_ids[(((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % (vocab_size * num_seq)) / vocab_size)] * vocab_size) + ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % vocab_size))] = condval; + } +} + +// Function: gpt_neox_q0f16_layer_norm1_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len) { + float in_thread_input_embeds_red_temp_v0_shared[1]; + float in_thread_input_embeds_red_temp_v1_shared[1]; + __local float red_buf0[256]; + __local float red_buf1[256]; + __local float input_embeds_red_temp_v0_shared[1]; + __local float input_embeds_red_temp_v1_shared[1]; + in_thread_input_embeds_red_temp_v0_shared[0] = 0.000000e+00f; + in_thread_input_embeds_red_temp_v1_shared[0] = 0.000000e+00f; + float v_input_embeds_red_temp_v0 = (in_thread_input_embeds_red_temp_v0_shared[0] + (convert_float(input_embeds[(((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0))))]))); + float v_input_embeds_red_temp_v1 = (in_thread_input_embeds_red_temp_v1_shared[0] + ((convert_float(input_embeds[(((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0))))])) * (convert_float(input_embeds[(((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0))))])))); + in_thread_input_embeds_red_temp_v0_shared[0] = v_input_embeds_red_temp_v0; + in_thread_input_embeds_red_temp_v1_shared[0] = v_input_embeds_red_temp_v1; + float v_input_embeds_red_temp_v0_1 = (in_thread_input_embeds_red_temp_v0_shared[0] + (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 256)]))); + float v_input_embeds_red_temp_v1_1 = (in_thread_input_embeds_red_temp_v1_shared[0] + ((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 256)])) * (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 256)])))); + in_thread_input_embeds_red_temp_v0_shared[0] = v_input_embeds_red_temp_v0_1; + in_thread_input_embeds_red_temp_v1_shared[0] = v_input_embeds_red_temp_v1_1; + float v_input_embeds_red_temp_v0_2 = (in_thread_input_embeds_red_temp_v0_shared[0] + (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 512)]))); + float v_input_embeds_red_temp_v1_2 = (in_thread_input_embeds_red_temp_v1_shared[0] + ((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 512)])) * (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 512)])))); + in_thread_input_embeds_red_temp_v0_shared[0] = v_input_embeds_red_temp_v0_2; + in_thread_input_embeds_red_temp_v1_shared[0] = v_input_embeds_red_temp_v1_2; + float v_input_embeds_red_temp_v0_3 = (in_thread_input_embeds_red_temp_v0_shared[0] + (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 768)]))); + float v_input_embeds_red_temp_v1_3 = (in_thread_input_embeds_red_temp_v1_shared[0] + ((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 768)])) * (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 768)])))); + in_thread_input_embeds_red_temp_v0_shared[0] = v_input_embeds_red_temp_v0_3; + in_thread_input_embeds_red_temp_v1_shared[0] = v_input_embeds_red_temp_v1_3; + float v_input_embeds_red_temp_v0_4 = (in_thread_input_embeds_red_temp_v0_shared[0] + (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1024)]))); + float v_input_embeds_red_temp_v1_4 = (in_thread_input_embeds_red_temp_v1_shared[0] + ((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1024)])) * (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1024)])))); + in_thread_input_embeds_red_temp_v0_shared[0] = v_input_embeds_red_temp_v0_4; + in_thread_input_embeds_red_temp_v1_shared[0] = v_input_embeds_red_temp_v1_4; + float v_input_embeds_red_temp_v0_5 = (in_thread_input_embeds_red_temp_v0_shared[0] + (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1280)]))); + float v_input_embeds_red_temp_v1_5 = (in_thread_input_embeds_red_temp_v1_shared[0] + ((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1280)])) * (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1280)])))); + in_thread_input_embeds_red_temp_v0_shared[0] = v_input_embeds_red_temp_v0_5; + in_thread_input_embeds_red_temp_v1_shared[0] = v_input_embeds_red_temp_v1_5; + float v_input_embeds_red_temp_v0_6 = (in_thread_input_embeds_red_temp_v0_shared[0] + (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1536)]))); + float v_input_embeds_red_temp_v1_6 = (in_thread_input_embeds_red_temp_v1_shared[0] + ((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1536)])) * (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1536)])))); + in_thread_input_embeds_red_temp_v0_shared[0] = v_input_embeds_red_temp_v0_6; + in_thread_input_embeds_red_temp_v1_shared[0] = v_input_embeds_red_temp_v1_6; + float v_input_embeds_red_temp_v0_7 = (in_thread_input_embeds_red_temp_v0_shared[0] + (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1792)]))); + float v_input_embeds_red_temp_v1_7 = (in_thread_input_embeds_red_temp_v1_shared[0] + ((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1792)])) * (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1792)])))); + in_thread_input_embeds_red_temp_v0_shared[0] = v_input_embeds_red_temp_v0_7; + in_thread_input_embeds_red_temp_v1_shared[0] = v_input_embeds_red_temp_v1_7; + barrier(CLK_LOCAL_MEM_FENCE); + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = in_thread_input_embeds_red_temp_v0_shared[0]; + ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = in_thread_input_embeds_red_temp_v1_shared[0]; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 128) { + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 128)]); + ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 128)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 64) { + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 64)]); + ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 64)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 32) { + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 32)]); + ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 32)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 16) { + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 16)]); + ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 16)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 8) { + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 8)]); + ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 8)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 4) { + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 4)]); + ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 4)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 2) { + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 2)]); + ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 2)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 1) { + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 1)]); + ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 1)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) == 0) { + input_embeds_red_temp_v0_shared[0] = ((volatile __local float*)red_buf0)[0]; + input_embeds_red_temp_v1_shared[0] = ((volatile __local float*)red_buf1)[0]; + } + barrier(CLK_LOCAL_MEM_FENCE); + T_layer_norm[(((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0))))] = (((convert_half((((convert_float(input_embeds[(((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0))))])) - (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embeds_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embeds_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight3[(convert_int(get_local_id(0)))]) + gpt_neox_layers_0_input_layernorm_bias3[(convert_int(get_local_id(0)))]); + T_layer_norm[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 256)] = (((convert_half((((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 256)])) - (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embeds_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embeds_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight3[((convert_int(get_local_id(0))) + 256)]) + gpt_neox_layers_0_input_layernorm_bias3[((convert_int(get_local_id(0))) + 256)]); + T_layer_norm[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 512)] = (((convert_half((((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 512)])) - (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embeds_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embeds_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight3[((convert_int(get_local_id(0))) + 512)]) + gpt_neox_layers_0_input_layernorm_bias3[((convert_int(get_local_id(0))) + 512)]); + T_layer_norm[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 768)] = (((convert_half((((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 768)])) - (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embeds_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embeds_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight3[((convert_int(get_local_id(0))) + 768)]) + gpt_neox_layers_0_input_layernorm_bias3[((convert_int(get_local_id(0))) + 768)]); + T_layer_norm[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1024)] = (((convert_half((((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1024)])) - (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embeds_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embeds_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight3[((convert_int(get_local_id(0))) + 1024)]) + gpt_neox_layers_0_input_layernorm_bias3[((convert_int(get_local_id(0))) + 1024)]); + T_layer_norm[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1280)] = (((convert_half((((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1280)])) - (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embeds_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embeds_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight3[((convert_int(get_local_id(0))) + 1280)]) + gpt_neox_layers_0_input_layernorm_bias3[((convert_int(get_local_id(0))) + 1280)]); + T_layer_norm[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1536)] = (((convert_half((((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1536)])) - (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embeds_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embeds_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight3[((convert_int(get_local_id(0))) + 1536)]) + gpt_neox_layers_0_input_layernorm_bias3[((convert_int(get_local_id(0))) + 1536)]); + T_layer_norm[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1792)] = (((convert_half((((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1792)])) - (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embeds_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embeds_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight3[((convert_int(get_local_id(0))) + 1792)]) + gpt_neox_layers_0_input_layernorm_bias3[((convert_int(get_local_id(0))) + 1792)]); +} + +// Function: gpt_neox_q0f16_fused_NT_matmul_add_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size) { + half NT_matmul_intermediate_pad_rf_local[32]; + half NT_matmul_intermediate_pad_rf_local_1[4]; + half NT_matmul_intermediate_pad_local[4]; + __local half red_buf0[64]; + for (int ax0_1_init = 0; ax0_1_init < 2; ++ax0_1_init) { + for (int ax1_fused_2_init = 0; ax1_fused_2_init < 2; ++ax1_fused_2_init) { + int8 v_ = (int8)((((ax0_1_init * 2) + ax1_fused_2_init))+(4*0), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*1), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*2), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*3), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*4), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*5), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*6), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*7)); + half8 v__1 = (half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f); + NT_matmul_intermediate_pad_rf_local[v_.s0] = v__1.s0; + NT_matmul_intermediate_pad_rf_local[v_.s1] = v__1.s1; + NT_matmul_intermediate_pad_rf_local[v_.s2] = v__1.s2; + NT_matmul_intermediate_pad_rf_local[v_.s3] = v__1.s3; + NT_matmul_intermediate_pad_rf_local[v_.s4] = v__1.s4; + NT_matmul_intermediate_pad_rf_local[v_.s5] = v__1.s5; + NT_matmul_intermediate_pad_rf_local[v_.s6] = v__1.s6; + NT_matmul_intermediate_pad_rf_local[v_.s7] = v__1.s7; + } + } + for (int ax2_fused_0 = 0; ax2_fused_0 < 8; ++ax2_fused_0) { + if (((convert_int(get_group_id(1))) * 2) < batch_size) { + int8 v__2 = (int8)((0)+(4*0), (0)+(4*1), (0)+(4*2), (0)+(4*3), (0)+(4*4), (0)+(4*5), (0)+(4*6), (0)+(4*7)); + half8 v__3 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__2.s0],NT_matmul_intermediate_pad_rf_local[v__2.s1],NT_matmul_intermediate_pad_rf_local[v__2.s2],NT_matmul_intermediate_pad_rf_local[v__2.s3],NT_matmul_intermediate_pad_rf_local[v__2.s4],NT_matmul_intermediate_pad_rf_local[v__2.s5],NT_matmul_intermediate_pad_rf_local[v__2.s6],NT_matmul_intermediate_pad_rf_local[v__2.s7])) + (vload8(0, layer_norm147 + ((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, gpt_neox_layers_0_attention_query_key_value_weight4 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))); + NT_matmul_intermediate_pad_rf_local[v__2.s0] = v__3.s0; + NT_matmul_intermediate_pad_rf_local[v__2.s1] = v__3.s1; + NT_matmul_intermediate_pad_rf_local[v__2.s2] = v__3.s2; + NT_matmul_intermediate_pad_rf_local[v__2.s3] = v__3.s3; + NT_matmul_intermediate_pad_rf_local[v__2.s4] = v__3.s4; + NT_matmul_intermediate_pad_rf_local[v__2.s5] = v__3.s5; + NT_matmul_intermediate_pad_rf_local[v__2.s6] = v__3.s6; + NT_matmul_intermediate_pad_rf_local[v__2.s7] = v__3.s7; + int8 v__4 = (int8)((1)+(4*0), (1)+(4*1), (1)+(4*2), (1)+(4*3), (1)+(4*4), (1)+(4*5), (1)+(4*6), (1)+(4*7)); + half8 v__5 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__4.s0],NT_matmul_intermediate_pad_rf_local[v__4.s1],NT_matmul_intermediate_pad_rf_local[v__4.s2],NT_matmul_intermediate_pad_rf_local[v__4.s3],NT_matmul_intermediate_pad_rf_local[v__4.s4],NT_matmul_intermediate_pad_rf_local[v__4.s5],NT_matmul_intermediate_pad_rf_local[v__4.s6],NT_matmul_intermediate_pad_rf_local[v__4.s7])) + (vload8(0, layer_norm147 + ((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, gpt_neox_layers_0_attention_query_key_value_weight4 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))); + NT_matmul_intermediate_pad_rf_local[v__4.s0] = v__5.s0; + NT_matmul_intermediate_pad_rf_local[v__4.s1] = v__5.s1; + NT_matmul_intermediate_pad_rf_local[v__4.s2] = v__5.s2; + NT_matmul_intermediate_pad_rf_local[v__4.s3] = v__5.s3; + NT_matmul_intermediate_pad_rf_local[v__4.s4] = v__5.s4; + NT_matmul_intermediate_pad_rf_local[v__4.s5] = v__5.s5; + NT_matmul_intermediate_pad_rf_local[v__4.s6] = v__5.s6; + NT_matmul_intermediate_pad_rf_local[v__4.s7] = v__5.s7; + } + if ((((convert_int(get_group_id(1))) * 2) + 1) < batch_size) { + int8 v__6 = (int8)((2)+(4*0), (2)+(4*1), (2)+(4*2), (2)+(4*3), (2)+(4*4), (2)+(4*5), (2)+(4*6), (2)+(4*7)); + half8 v__7 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__6.s0],NT_matmul_intermediate_pad_rf_local[v__6.s1],NT_matmul_intermediate_pad_rf_local[v__6.s2],NT_matmul_intermediate_pad_rf_local[v__6.s3],NT_matmul_intermediate_pad_rf_local[v__6.s4],NT_matmul_intermediate_pad_rf_local[v__6.s5],NT_matmul_intermediate_pad_rf_local[v__6.s6],NT_matmul_intermediate_pad_rf_local[v__6.s7])) + (vload8(0, layer_norm147 + (((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)) * vload8(0, gpt_neox_layers_0_attention_query_key_value_weight4 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))); + NT_matmul_intermediate_pad_rf_local[v__6.s0] = v__7.s0; + NT_matmul_intermediate_pad_rf_local[v__6.s1] = v__7.s1; + NT_matmul_intermediate_pad_rf_local[v__6.s2] = v__7.s2; + NT_matmul_intermediate_pad_rf_local[v__6.s3] = v__7.s3; + NT_matmul_intermediate_pad_rf_local[v__6.s4] = v__7.s4; + NT_matmul_intermediate_pad_rf_local[v__6.s5] = v__7.s5; + NT_matmul_intermediate_pad_rf_local[v__6.s6] = v__7.s6; + NT_matmul_intermediate_pad_rf_local[v__6.s7] = v__7.s7; + int8 v__8 = (int8)((3)+(4*0), (3)+(4*1), (3)+(4*2), (3)+(4*3), (3)+(4*4), (3)+(4*5), (3)+(4*6), (3)+(4*7)); + half8 v__9 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__8.s0],NT_matmul_intermediate_pad_rf_local[v__8.s1],NT_matmul_intermediate_pad_rf_local[v__8.s2],NT_matmul_intermediate_pad_rf_local[v__8.s3],NT_matmul_intermediate_pad_rf_local[v__8.s4],NT_matmul_intermediate_pad_rf_local[v__8.s5],NT_matmul_intermediate_pad_rf_local[v__8.s6],NT_matmul_intermediate_pad_rf_local[v__8.s7])) + (vload8(0, layer_norm147 + (((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)) * vload8(0, gpt_neox_layers_0_attention_query_key_value_weight4 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))); + NT_matmul_intermediate_pad_rf_local[v__8.s0] = v__9.s0; + NT_matmul_intermediate_pad_rf_local[v__8.s1] = v__9.s1; + NT_matmul_intermediate_pad_rf_local[v__8.s2] = v__9.s2; + NT_matmul_intermediate_pad_rf_local[v__8.s3] = v__9.s3; + NT_matmul_intermediate_pad_rf_local[v__8.s4] = v__9.s4; + NT_matmul_intermediate_pad_rf_local[v__8.s5] = v__9.s5; + NT_matmul_intermediate_pad_rf_local[v__8.s6] = v__9.s6; + NT_matmul_intermediate_pad_rf_local[v__8.s7] = v__9.s7; + } + } + for (int ax3_fused_2_0 = 0; ax3_fused_2_0 < 1; ++ax3_fused_2_0) { + for (int ax2 = 0; ax2 < 2; ++ax2) { + vstore2(((half2)((half)0.000000e+00f, (half)0.000000e+00f)), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + (ax2 * 2))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 4))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 8))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 12))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 16))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 20))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 24))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 28))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + } + } + for (int ax2_fused_2 = 0; ax2_fused_2 < 2; ++ax2_fused_2) { + for (int ax1 = 0; ax1 < 2; ++ax1) { + barrier(CLK_LOCAL_MEM_FENCE); + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_intermediate_pad_rf_local_1[((ax1 * 2) + ax2_fused_2)]; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 16) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 8) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 4) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 2) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + NT_matmul_intermediate_pad_local[((ax1 * 2) + ax2_fused_2)] = ((volatile __local half*)red_buf0)[((convert_int(get_local_id(0))) * 32)]; + } + } + for (int ax0 = 0; ax0 < 2; ++ax0) { + for (int ax1_fused_2 = 0; ax1_fused_2 < 2; ++ax1_fused_2) { + if (((bool)(((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)(((bool)(((convert_int(get_group_id(1))) - ((batch_size + 1) >> 1)) < 0)) || ((bool)((convert_int(get_group_id(1))) == 0)))))) && ((bool)((((convert_int(get_group_id(1))) * 2) + ax0) < batch_size))) { + T_add[((((((convert_int(get_group_id(1))) * 12288) + (ax0 * 6144)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] = (NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_attention_query_key_value_bias4[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]); + } + } + } +} + +// Function: gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len) { + float NT_matmul_intermediate_pad_rf_local[32]; + float NT_matmul_intermediate_pad_rf_local_1[4]; + float NT_matmul_intermediate_pad_local[4]; + __local float red_buf0[64]; + for (int ax0_1_init = 0; ax0_1_init < 2; ++ax0_1_init) { + for (int ax1_fused_2_init = 0; ax1_fused_2_init < 2; ++ax1_fused_2_init) { + int8 v_ = (int8)((((ax0_1_init * 2) + ax1_fused_2_init))+(4*0), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*1), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*2), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*3), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*4), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*5), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*6), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*7)); + float8 v__1 = (float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f); + NT_matmul_intermediate_pad_rf_local[v_.s0] = v__1.s0; + NT_matmul_intermediate_pad_rf_local[v_.s1] = v__1.s1; + NT_matmul_intermediate_pad_rf_local[v_.s2] = v__1.s2; + NT_matmul_intermediate_pad_rf_local[v_.s3] = v__1.s3; + NT_matmul_intermediate_pad_rf_local[v_.s4] = v__1.s4; + NT_matmul_intermediate_pad_rf_local[v_.s5] = v__1.s5; + NT_matmul_intermediate_pad_rf_local[v_.s6] = v__1.s6; + NT_matmul_intermediate_pad_rf_local[v_.s7] = v__1.s7; + } + } + for (int ax2_fused_0 = 0; ax2_fused_0 < 8; ++ax2_fused_0) { + if (((convert_int(get_group_id(1))) * 2) < seq_len) { + int8 v__2 = (int8)((0)+(4*0), (0)+(4*1), (0)+(4*2), (0)+(4*3), (0)+(4*4), (0)+(4*5), (0)+(4*6), (0)+(4*7)); + float8 v__3 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__2.s0],NT_matmul_intermediate_pad_rf_local[v__2.s1],NT_matmul_intermediate_pad_rf_local[v__2.s2],NT_matmul_intermediate_pad_rf_local[v__2.s3],NT_matmul_intermediate_pad_rf_local[v__2.s4],NT_matmul_intermediate_pad_rf_local[v__2.s5],NT_matmul_intermediate_pad_rf_local[v__2.s6],NT_matmul_intermediate_pad_rf_local[v__2.s7])) + ((convert_float8(vload8(0, layer_norm99 + ((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_h_to_4h_weight3 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))))); + NT_matmul_intermediate_pad_rf_local[v__2.s0] = v__3.s0; + NT_matmul_intermediate_pad_rf_local[v__2.s1] = v__3.s1; + NT_matmul_intermediate_pad_rf_local[v__2.s2] = v__3.s2; + NT_matmul_intermediate_pad_rf_local[v__2.s3] = v__3.s3; + NT_matmul_intermediate_pad_rf_local[v__2.s4] = v__3.s4; + NT_matmul_intermediate_pad_rf_local[v__2.s5] = v__3.s5; + NT_matmul_intermediate_pad_rf_local[v__2.s6] = v__3.s6; + NT_matmul_intermediate_pad_rf_local[v__2.s7] = v__3.s7; + int8 v__4 = (int8)((1)+(4*0), (1)+(4*1), (1)+(4*2), (1)+(4*3), (1)+(4*4), (1)+(4*5), (1)+(4*6), (1)+(4*7)); + float8 v__5 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__4.s0],NT_matmul_intermediate_pad_rf_local[v__4.s1],NT_matmul_intermediate_pad_rf_local[v__4.s2],NT_matmul_intermediate_pad_rf_local[v__4.s3],NT_matmul_intermediate_pad_rf_local[v__4.s4],NT_matmul_intermediate_pad_rf_local[v__4.s5],NT_matmul_intermediate_pad_rf_local[v__4.s6],NT_matmul_intermediate_pad_rf_local[v__4.s7])) + ((convert_float8(vload8(0, layer_norm99 + ((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_h_to_4h_weight3 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))))); + NT_matmul_intermediate_pad_rf_local[v__4.s0] = v__5.s0; + NT_matmul_intermediate_pad_rf_local[v__4.s1] = v__5.s1; + NT_matmul_intermediate_pad_rf_local[v__4.s2] = v__5.s2; + NT_matmul_intermediate_pad_rf_local[v__4.s3] = v__5.s3; + NT_matmul_intermediate_pad_rf_local[v__4.s4] = v__5.s4; + NT_matmul_intermediate_pad_rf_local[v__4.s5] = v__5.s5; + NT_matmul_intermediate_pad_rf_local[v__4.s6] = v__5.s6; + NT_matmul_intermediate_pad_rf_local[v__4.s7] = v__5.s7; + } + if ((((convert_int(get_group_id(1))) * 2) + 1) < seq_len) { + int8 v__6 = (int8)((2)+(4*0), (2)+(4*1), (2)+(4*2), (2)+(4*3), (2)+(4*4), (2)+(4*5), (2)+(4*6), (2)+(4*7)); + float8 v__7 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__6.s0],NT_matmul_intermediate_pad_rf_local[v__6.s1],NT_matmul_intermediate_pad_rf_local[v__6.s2],NT_matmul_intermediate_pad_rf_local[v__6.s3],NT_matmul_intermediate_pad_rf_local[v__6.s4],NT_matmul_intermediate_pad_rf_local[v__6.s5],NT_matmul_intermediate_pad_rf_local[v__6.s6],NT_matmul_intermediate_pad_rf_local[v__6.s7])) + ((convert_float8(vload8(0, layer_norm99 + (((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_h_to_4h_weight3 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))))); + NT_matmul_intermediate_pad_rf_local[v__6.s0] = v__7.s0; + NT_matmul_intermediate_pad_rf_local[v__6.s1] = v__7.s1; + NT_matmul_intermediate_pad_rf_local[v__6.s2] = v__7.s2; + NT_matmul_intermediate_pad_rf_local[v__6.s3] = v__7.s3; + NT_matmul_intermediate_pad_rf_local[v__6.s4] = v__7.s4; + NT_matmul_intermediate_pad_rf_local[v__6.s5] = v__7.s5; + NT_matmul_intermediate_pad_rf_local[v__6.s6] = v__7.s6; + NT_matmul_intermediate_pad_rf_local[v__6.s7] = v__7.s7; + int8 v__8 = (int8)((3)+(4*0), (3)+(4*1), (3)+(4*2), (3)+(4*3), (3)+(4*4), (3)+(4*5), (3)+(4*6), (3)+(4*7)); + float8 v__9 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__8.s0],NT_matmul_intermediate_pad_rf_local[v__8.s1],NT_matmul_intermediate_pad_rf_local[v__8.s2],NT_matmul_intermediate_pad_rf_local[v__8.s3],NT_matmul_intermediate_pad_rf_local[v__8.s4],NT_matmul_intermediate_pad_rf_local[v__8.s5],NT_matmul_intermediate_pad_rf_local[v__8.s6],NT_matmul_intermediate_pad_rf_local[v__8.s7])) + ((convert_float8(vload8(0, layer_norm99 + (((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_h_to_4h_weight3 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))))); + NT_matmul_intermediate_pad_rf_local[v__8.s0] = v__9.s0; + NT_matmul_intermediate_pad_rf_local[v__8.s1] = v__9.s1; + NT_matmul_intermediate_pad_rf_local[v__8.s2] = v__9.s2; + NT_matmul_intermediate_pad_rf_local[v__8.s3] = v__9.s3; + NT_matmul_intermediate_pad_rf_local[v__8.s4] = v__9.s4; + NT_matmul_intermediate_pad_rf_local[v__8.s5] = v__9.s5; + NT_matmul_intermediate_pad_rf_local[v__8.s6] = v__9.s6; + NT_matmul_intermediate_pad_rf_local[v__8.s7] = v__9.s7; + } + } + for (int ax3_fused_2_0 = 0; ax3_fused_2_0 < 1; ++ax3_fused_2_0) { + for (int ax2 = 0; ax2 < 2; ++ax2) { + vstore2(((float2)(0.000000e+00f, 0.000000e+00f)), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + (ax2 * 2))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 4))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 8))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 12))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 16))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 20))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 24))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 28))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + } + } + for (int ax2_fused_2 = 0; ax2_fused_2 < 2; ++ax2_fused_2) { + for (int ax1 = 0; ax1 < 2; ++ax1) { + barrier(CLK_LOCAL_MEM_FENCE); + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_intermediate_pad_rf_local_1[((ax1 * 2) + ax2_fused_2)]; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 16) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 8) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 4) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 2) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + NT_matmul_intermediate_pad_local[((ax1 * 2) + ax2_fused_2)] = ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) * 32)]; + } + } + for (int ax0 = 0; ax0 < 2; ++ax0) { + for (int ax1_fused_2 = 0; ax1_fused_2 < 2; ++ax1_fused_2) { + if (((bool)(((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)(((bool)(((convert_int(get_group_id(1))) - ((seq_len + 1) >> 1)) < 0)) || ((bool)((convert_int(get_group_id(1))) == 0)))))) && ((bool)((((convert_int(get_group_id(1))) * 2) + ax0) < seq_len))) { + compute[((((((convert_int(get_group_id(1))) * 16384) + (ax0 * 8192)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] = (convert_half(((NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]) * 7.071068e-01f)) * 5.000000e-01f))))); + } + } + } +} + +// Function: gpt_neox_q0f16_fused_reshape10_reshape11_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25) { + T_reshape[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] = lv25[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]; +} + +// Function: gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1 +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len) { + half NT_matmul_intermediate_pad_rf_local[64]; + half NT_matmul_intermediate_pad_rf_local_1[8]; + half NT_matmul_intermediate_pad_local[8]; + __local half red_buf0[64]; + for (int ax0_1_init = 0; ax0_1_init < 4; ++ax0_1_init) { + for (int ax1_fused_2_init = 0; ax1_fused_2_init < 2; ++ax1_fused_2_init) { + int8 v_ = (int8)((((ax0_1_init * 2) + ax1_fused_2_init))+(8*0), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*1), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*2), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*3), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*4), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*5), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*6), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*7)); + half8 v__1 = (half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f); + NT_matmul_intermediate_pad_rf_local[v_.s0] = v__1.s0; + NT_matmul_intermediate_pad_rf_local[v_.s1] = v__1.s1; + NT_matmul_intermediate_pad_rf_local[v_.s2] = v__1.s2; + NT_matmul_intermediate_pad_rf_local[v_.s3] = v__1.s3; + NT_matmul_intermediate_pad_rf_local[v_.s4] = v__1.s4; + NT_matmul_intermediate_pad_rf_local[v_.s5] = v__1.s5; + NT_matmul_intermediate_pad_rf_local[v_.s6] = v__1.s6; + NT_matmul_intermediate_pad_rf_local[v_.s7] = v__1.s7; + } + } + for (int ax2_fused_0 = 0; ax2_fused_0 < 8; ++ax2_fused_0) { + if (((convert_int(get_group_id(1))) * 4) < seq_len) { + int8 v__2 = (int8)((0)+(8*0), (0)+(8*1), (0)+(8*2), (0)+(8*3), (0)+(8*4), (0)+(8*5), (0)+(8*6), (0)+(8*7)); + half8 v__3 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__2.s0],NT_matmul_intermediate_pad_rf_local[v__2.s1],NT_matmul_intermediate_pad_rf_local[v__2.s2],NT_matmul_intermediate_pad_rf_local[v__2.s3],NT_matmul_intermediate_pad_rf_local[v__2.s4],NT_matmul_intermediate_pad_rf_local[v__2.s5],NT_matmul_intermediate_pad_rf_local[v__2.s6],NT_matmul_intermediate_pad_rf_local[v__2.s7])) + (vload8(0, reshape195 + ((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, gpt_neox_layers_0_attention_dense_weight3 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))); + NT_matmul_intermediate_pad_rf_local[v__2.s0] = v__3.s0; + NT_matmul_intermediate_pad_rf_local[v__2.s1] = v__3.s1; + NT_matmul_intermediate_pad_rf_local[v__2.s2] = v__3.s2; + NT_matmul_intermediate_pad_rf_local[v__2.s3] = v__3.s3; + NT_matmul_intermediate_pad_rf_local[v__2.s4] = v__3.s4; + NT_matmul_intermediate_pad_rf_local[v__2.s5] = v__3.s5; + NT_matmul_intermediate_pad_rf_local[v__2.s6] = v__3.s6; + NT_matmul_intermediate_pad_rf_local[v__2.s7] = v__3.s7; + int8 v__4 = (int8)((1)+(8*0), (1)+(8*1), (1)+(8*2), (1)+(8*3), (1)+(8*4), (1)+(8*5), (1)+(8*6), (1)+(8*7)); + half8 v__5 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__4.s0],NT_matmul_intermediate_pad_rf_local[v__4.s1],NT_matmul_intermediate_pad_rf_local[v__4.s2],NT_matmul_intermediate_pad_rf_local[v__4.s3],NT_matmul_intermediate_pad_rf_local[v__4.s4],NT_matmul_intermediate_pad_rf_local[v__4.s5],NT_matmul_intermediate_pad_rf_local[v__4.s6],NT_matmul_intermediate_pad_rf_local[v__4.s7])) + (vload8(0, reshape195 + ((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, gpt_neox_layers_0_attention_dense_weight3 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))); + NT_matmul_intermediate_pad_rf_local[v__4.s0] = v__5.s0; + NT_matmul_intermediate_pad_rf_local[v__4.s1] = v__5.s1; + NT_matmul_intermediate_pad_rf_local[v__4.s2] = v__5.s2; + NT_matmul_intermediate_pad_rf_local[v__4.s3] = v__5.s3; + NT_matmul_intermediate_pad_rf_local[v__4.s4] = v__5.s4; + NT_matmul_intermediate_pad_rf_local[v__4.s5] = v__5.s5; + NT_matmul_intermediate_pad_rf_local[v__4.s6] = v__5.s6; + NT_matmul_intermediate_pad_rf_local[v__4.s7] = v__5.s7; + } + if ((((convert_int(get_group_id(1))) * 4) + 1) < seq_len) { + int8 v__6 = (int8)((2)+(8*0), (2)+(8*1), (2)+(8*2), (2)+(8*3), (2)+(8*4), (2)+(8*5), (2)+(8*6), (2)+(8*7)); + half8 v__7 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__6.s0],NT_matmul_intermediate_pad_rf_local[v__6.s1],NT_matmul_intermediate_pad_rf_local[v__6.s2],NT_matmul_intermediate_pad_rf_local[v__6.s3],NT_matmul_intermediate_pad_rf_local[v__6.s4],NT_matmul_intermediate_pad_rf_local[v__6.s5],NT_matmul_intermediate_pad_rf_local[v__6.s6],NT_matmul_intermediate_pad_rf_local[v__6.s7])) + (vload8(0, reshape195 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)) * vload8(0, gpt_neox_layers_0_attention_dense_weight3 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))); + NT_matmul_intermediate_pad_rf_local[v__6.s0] = v__7.s0; + NT_matmul_intermediate_pad_rf_local[v__6.s1] = v__7.s1; + NT_matmul_intermediate_pad_rf_local[v__6.s2] = v__7.s2; + NT_matmul_intermediate_pad_rf_local[v__6.s3] = v__7.s3; + NT_matmul_intermediate_pad_rf_local[v__6.s4] = v__7.s4; + NT_matmul_intermediate_pad_rf_local[v__6.s5] = v__7.s5; + NT_matmul_intermediate_pad_rf_local[v__6.s6] = v__7.s6; + NT_matmul_intermediate_pad_rf_local[v__6.s7] = v__7.s7; + int8 v__8 = (int8)((3)+(8*0), (3)+(8*1), (3)+(8*2), (3)+(8*3), (3)+(8*4), (3)+(8*5), (3)+(8*6), (3)+(8*7)); + half8 v__9 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__8.s0],NT_matmul_intermediate_pad_rf_local[v__8.s1],NT_matmul_intermediate_pad_rf_local[v__8.s2],NT_matmul_intermediate_pad_rf_local[v__8.s3],NT_matmul_intermediate_pad_rf_local[v__8.s4],NT_matmul_intermediate_pad_rf_local[v__8.s5],NT_matmul_intermediate_pad_rf_local[v__8.s6],NT_matmul_intermediate_pad_rf_local[v__8.s7])) + (vload8(0, reshape195 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)) * vload8(0, gpt_neox_layers_0_attention_dense_weight3 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))); + NT_matmul_intermediate_pad_rf_local[v__8.s0] = v__9.s0; + NT_matmul_intermediate_pad_rf_local[v__8.s1] = v__9.s1; + NT_matmul_intermediate_pad_rf_local[v__8.s2] = v__9.s2; + NT_matmul_intermediate_pad_rf_local[v__8.s3] = v__9.s3; + NT_matmul_intermediate_pad_rf_local[v__8.s4] = v__9.s4; + NT_matmul_intermediate_pad_rf_local[v__8.s5] = v__9.s5; + NT_matmul_intermediate_pad_rf_local[v__8.s6] = v__9.s6; + NT_matmul_intermediate_pad_rf_local[v__8.s7] = v__9.s7; + } + if ((((convert_int(get_group_id(1))) * 4) + 2) < seq_len) { + int8 v__10 = (int8)((4)+(8*0), (4)+(8*1), (4)+(8*2), (4)+(8*3), (4)+(8*4), (4)+(8*5), (4)+(8*6), (4)+(8*7)); + half8 v__11 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__10.s0],NT_matmul_intermediate_pad_rf_local[v__10.s1],NT_matmul_intermediate_pad_rf_local[v__10.s2],NT_matmul_intermediate_pad_rf_local[v__10.s3],NT_matmul_intermediate_pad_rf_local[v__10.s4],NT_matmul_intermediate_pad_rf_local[v__10.s5],NT_matmul_intermediate_pad_rf_local[v__10.s6],NT_matmul_intermediate_pad_rf_local[v__10.s7])) + (vload8(0, reshape195 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 4096)) * vload8(0, gpt_neox_layers_0_attention_dense_weight3 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))); + NT_matmul_intermediate_pad_rf_local[v__10.s0] = v__11.s0; + NT_matmul_intermediate_pad_rf_local[v__10.s1] = v__11.s1; + NT_matmul_intermediate_pad_rf_local[v__10.s2] = v__11.s2; + NT_matmul_intermediate_pad_rf_local[v__10.s3] = v__11.s3; + NT_matmul_intermediate_pad_rf_local[v__10.s4] = v__11.s4; + NT_matmul_intermediate_pad_rf_local[v__10.s5] = v__11.s5; + NT_matmul_intermediate_pad_rf_local[v__10.s6] = v__11.s6; + NT_matmul_intermediate_pad_rf_local[v__10.s7] = v__11.s7; + int8 v__12 = (int8)((5)+(8*0), (5)+(8*1), (5)+(8*2), (5)+(8*3), (5)+(8*4), (5)+(8*5), (5)+(8*6), (5)+(8*7)); + half8 v__13 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__12.s0],NT_matmul_intermediate_pad_rf_local[v__12.s1],NT_matmul_intermediate_pad_rf_local[v__12.s2],NT_matmul_intermediate_pad_rf_local[v__12.s3],NT_matmul_intermediate_pad_rf_local[v__12.s4],NT_matmul_intermediate_pad_rf_local[v__12.s5],NT_matmul_intermediate_pad_rf_local[v__12.s6],NT_matmul_intermediate_pad_rf_local[v__12.s7])) + (vload8(0, reshape195 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 4096)) * vload8(0, gpt_neox_layers_0_attention_dense_weight3 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))); + NT_matmul_intermediate_pad_rf_local[v__12.s0] = v__13.s0; + NT_matmul_intermediate_pad_rf_local[v__12.s1] = v__13.s1; + NT_matmul_intermediate_pad_rf_local[v__12.s2] = v__13.s2; + NT_matmul_intermediate_pad_rf_local[v__12.s3] = v__13.s3; + NT_matmul_intermediate_pad_rf_local[v__12.s4] = v__13.s4; + NT_matmul_intermediate_pad_rf_local[v__12.s5] = v__13.s5; + NT_matmul_intermediate_pad_rf_local[v__12.s6] = v__13.s6; + NT_matmul_intermediate_pad_rf_local[v__12.s7] = v__13.s7; + } + if ((((convert_int(get_group_id(1))) * 4) + 3) < seq_len) { + int8 v__14 = (int8)((6)+(8*0), (6)+(8*1), (6)+(8*2), (6)+(8*3), (6)+(8*4), (6)+(8*5), (6)+(8*6), (6)+(8*7)); + half8 v__15 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__14.s0],NT_matmul_intermediate_pad_rf_local[v__14.s1],NT_matmul_intermediate_pad_rf_local[v__14.s2],NT_matmul_intermediate_pad_rf_local[v__14.s3],NT_matmul_intermediate_pad_rf_local[v__14.s4],NT_matmul_intermediate_pad_rf_local[v__14.s5],NT_matmul_intermediate_pad_rf_local[v__14.s6],NT_matmul_intermediate_pad_rf_local[v__14.s7])) + (vload8(0, reshape195 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 6144)) * vload8(0, gpt_neox_layers_0_attention_dense_weight3 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))); + NT_matmul_intermediate_pad_rf_local[v__14.s0] = v__15.s0; + NT_matmul_intermediate_pad_rf_local[v__14.s1] = v__15.s1; + NT_matmul_intermediate_pad_rf_local[v__14.s2] = v__15.s2; + NT_matmul_intermediate_pad_rf_local[v__14.s3] = v__15.s3; + NT_matmul_intermediate_pad_rf_local[v__14.s4] = v__15.s4; + NT_matmul_intermediate_pad_rf_local[v__14.s5] = v__15.s5; + NT_matmul_intermediate_pad_rf_local[v__14.s6] = v__15.s6; + NT_matmul_intermediate_pad_rf_local[v__14.s7] = v__15.s7; + int8 v__16 = (int8)((7)+(8*0), (7)+(8*1), (7)+(8*2), (7)+(8*3), (7)+(8*4), (7)+(8*5), (7)+(8*6), (7)+(8*7)); + half8 v__17 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__16.s0],NT_matmul_intermediate_pad_rf_local[v__16.s1],NT_matmul_intermediate_pad_rf_local[v__16.s2],NT_matmul_intermediate_pad_rf_local[v__16.s3],NT_matmul_intermediate_pad_rf_local[v__16.s4],NT_matmul_intermediate_pad_rf_local[v__16.s5],NT_matmul_intermediate_pad_rf_local[v__16.s6],NT_matmul_intermediate_pad_rf_local[v__16.s7])) + (vload8(0, reshape195 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 6144)) * vload8(0, gpt_neox_layers_0_attention_dense_weight3 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))); + NT_matmul_intermediate_pad_rf_local[v__16.s0] = v__17.s0; + NT_matmul_intermediate_pad_rf_local[v__16.s1] = v__17.s1; + NT_matmul_intermediate_pad_rf_local[v__16.s2] = v__17.s2; + NT_matmul_intermediate_pad_rf_local[v__16.s3] = v__17.s3; + NT_matmul_intermediate_pad_rf_local[v__16.s4] = v__17.s4; + NT_matmul_intermediate_pad_rf_local[v__16.s5] = v__17.s5; + NT_matmul_intermediate_pad_rf_local[v__16.s6] = v__17.s6; + NT_matmul_intermediate_pad_rf_local[v__16.s7] = v__17.s7; + } + } + for (int ax3_fused_2_0 = 0; ax3_fused_2_0 < 1; ++ax3_fused_2_0) { + for (int ax2 = 0; ax2 < 4; ++ax2) { + vstore2(((half2)((half)0.000000e+00f, (half)0.000000e+00f)), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + (ax2 * 2))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 8))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 16))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 24))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 32))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 40))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 48))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 56))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + } + } + for (int ax2_fused_2 = 0; ax2_fused_2 < 2; ++ax2_fused_2) { + for (int ax1 = 0; ax1 < 4; ++ax1) { + barrier(CLK_LOCAL_MEM_FENCE); + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_intermediate_pad_rf_local_1[((ax1 * 2) + ax2_fused_2)]; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 16) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 8) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 4) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 2) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + NT_matmul_intermediate_pad_local[((ax1 * 2) + ax2_fused_2)] = ((volatile __local half*)red_buf0)[((convert_int(get_local_id(0))) * 32)]; + } + } + for (int ax0 = 0; ax0 < 4; ++ax0) { + for (int ax1_fused_2 = 0; ax1_fused_2 < 2; ++ax1_fused_2) { + if (((bool)(((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)(((bool)(((convert_int(get_group_id(1))) - ((seq_len + 3) >> 2)) < 0)) || ((bool)((convert_int(get_group_id(1))) == 0)))))) && ((bool)((((convert_int(get_group_id(1))) * 4) + ax0) < seq_len))) { + T_add[((((((convert_int(get_group_id(1))) * 8192) + (ax0 * 2048)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] = ((astype99[((((((convert_int(get_group_id(1))) * 8192) + (ax0 * 2048)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] + (NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_attention_dense_bias3[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)])) + input_embeds[((((((convert_int(get_group_id(1))) * 8192) + (ax0 * 2048)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]); + } + } + } +} + +// Function: gpt_neox_q0f16_reshape1_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size) { + T_reshape[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] = reshape288[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]; +} + +// Function: gpt_neox_q0f16_reshape3_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size) { + T_reshape[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] = reshape290[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]; +} + +// Function: gpt_neox_q0f16_reshape4_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len) { + T_reshape[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] = add288[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]; +} + +// Function: gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1 +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size) { + float NT_matmul_intermediate_pad_rf_local[64]; + float NT_matmul_intermediate_pad_rf_local_1[8]; + float NT_matmul_intermediate_pad_local[8]; + __local float red_buf0[64]; + for (int ax0_1_init = 0; ax0_1_init < 4; ++ax0_1_init) { + for (int ax1_fused_2_init = 0; ax1_fused_2_init < 2; ++ax1_fused_2_init) { + int8 v_ = (int8)((((ax0_1_init * 2) + ax1_fused_2_init))+(8*0), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*1), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*2), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*3), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*4), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*5), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*6), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*7)); + float8 v__1 = (float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f); + NT_matmul_intermediate_pad_rf_local[v_.s0] = v__1.s0; + NT_matmul_intermediate_pad_rf_local[v_.s1] = v__1.s1; + NT_matmul_intermediate_pad_rf_local[v_.s2] = v__1.s2; + NT_matmul_intermediate_pad_rf_local[v_.s3] = v__1.s3; + NT_matmul_intermediate_pad_rf_local[v_.s4] = v__1.s4; + NT_matmul_intermediate_pad_rf_local[v_.s5] = v__1.s5; + NT_matmul_intermediate_pad_rf_local[v_.s6] = v__1.s6; + NT_matmul_intermediate_pad_rf_local[v_.s7] = v__1.s7; + } + } + for (int ax2_fused_0 = 0; ax2_fused_0 < 8; ++ax2_fused_0) { + if (((convert_int(get_group_id(1))) * 4) < batch_size) { + int8 v__2 = (int8)((0)+(8*0), (0)+(8*1), (0)+(8*2), (0)+(8*3), (0)+(8*4), (0)+(8*5), (0)+(8*6), (0)+(8*7)); + float8 v__3 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__2.s0],NT_matmul_intermediate_pad_rf_local[v__2.s1],NT_matmul_intermediate_pad_rf_local[v__2.s2],NT_matmul_intermediate_pad_rf_local[v__2.s3],NT_matmul_intermediate_pad_rf_local[v__2.s4],NT_matmul_intermediate_pad_rf_local[v__2.s5],NT_matmul_intermediate_pad_rf_local[v__2.s6],NT_matmul_intermediate_pad_rf_local[v__2.s7])) + ((convert_float8(vload8(0, layer_norm148 + ((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_h_to_4h_weight4 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))))); + NT_matmul_intermediate_pad_rf_local[v__2.s0] = v__3.s0; + NT_matmul_intermediate_pad_rf_local[v__2.s1] = v__3.s1; + NT_matmul_intermediate_pad_rf_local[v__2.s2] = v__3.s2; + NT_matmul_intermediate_pad_rf_local[v__2.s3] = v__3.s3; + NT_matmul_intermediate_pad_rf_local[v__2.s4] = v__3.s4; + NT_matmul_intermediate_pad_rf_local[v__2.s5] = v__3.s5; + NT_matmul_intermediate_pad_rf_local[v__2.s6] = v__3.s6; + NT_matmul_intermediate_pad_rf_local[v__2.s7] = v__3.s7; + int8 v__4 = (int8)((1)+(8*0), (1)+(8*1), (1)+(8*2), (1)+(8*3), (1)+(8*4), (1)+(8*5), (1)+(8*6), (1)+(8*7)); + float8 v__5 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__4.s0],NT_matmul_intermediate_pad_rf_local[v__4.s1],NT_matmul_intermediate_pad_rf_local[v__4.s2],NT_matmul_intermediate_pad_rf_local[v__4.s3],NT_matmul_intermediate_pad_rf_local[v__4.s4],NT_matmul_intermediate_pad_rf_local[v__4.s5],NT_matmul_intermediate_pad_rf_local[v__4.s6],NT_matmul_intermediate_pad_rf_local[v__4.s7])) + ((convert_float8(vload8(0, layer_norm148 + ((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_h_to_4h_weight4 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))))); + NT_matmul_intermediate_pad_rf_local[v__4.s0] = v__5.s0; + NT_matmul_intermediate_pad_rf_local[v__4.s1] = v__5.s1; + NT_matmul_intermediate_pad_rf_local[v__4.s2] = v__5.s2; + NT_matmul_intermediate_pad_rf_local[v__4.s3] = v__5.s3; + NT_matmul_intermediate_pad_rf_local[v__4.s4] = v__5.s4; + NT_matmul_intermediate_pad_rf_local[v__4.s5] = v__5.s5; + NT_matmul_intermediate_pad_rf_local[v__4.s6] = v__5.s6; + NT_matmul_intermediate_pad_rf_local[v__4.s7] = v__5.s7; + } + if ((((convert_int(get_group_id(1))) * 4) + 1) < batch_size) { + int8 v__6 = (int8)((2)+(8*0), (2)+(8*1), (2)+(8*2), (2)+(8*3), (2)+(8*4), (2)+(8*5), (2)+(8*6), (2)+(8*7)); + float8 v__7 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__6.s0],NT_matmul_intermediate_pad_rf_local[v__6.s1],NT_matmul_intermediate_pad_rf_local[v__6.s2],NT_matmul_intermediate_pad_rf_local[v__6.s3],NT_matmul_intermediate_pad_rf_local[v__6.s4],NT_matmul_intermediate_pad_rf_local[v__6.s5],NT_matmul_intermediate_pad_rf_local[v__6.s6],NT_matmul_intermediate_pad_rf_local[v__6.s7])) + ((convert_float8(vload8(0, layer_norm148 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_h_to_4h_weight4 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))))); + NT_matmul_intermediate_pad_rf_local[v__6.s0] = v__7.s0; + NT_matmul_intermediate_pad_rf_local[v__6.s1] = v__7.s1; + NT_matmul_intermediate_pad_rf_local[v__6.s2] = v__7.s2; + NT_matmul_intermediate_pad_rf_local[v__6.s3] = v__7.s3; + NT_matmul_intermediate_pad_rf_local[v__6.s4] = v__7.s4; + NT_matmul_intermediate_pad_rf_local[v__6.s5] = v__7.s5; + NT_matmul_intermediate_pad_rf_local[v__6.s6] = v__7.s6; + NT_matmul_intermediate_pad_rf_local[v__6.s7] = v__7.s7; + int8 v__8 = (int8)((3)+(8*0), (3)+(8*1), (3)+(8*2), (3)+(8*3), (3)+(8*4), (3)+(8*5), (3)+(8*6), (3)+(8*7)); + float8 v__9 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__8.s0],NT_matmul_intermediate_pad_rf_local[v__8.s1],NT_matmul_intermediate_pad_rf_local[v__8.s2],NT_matmul_intermediate_pad_rf_local[v__8.s3],NT_matmul_intermediate_pad_rf_local[v__8.s4],NT_matmul_intermediate_pad_rf_local[v__8.s5],NT_matmul_intermediate_pad_rf_local[v__8.s6],NT_matmul_intermediate_pad_rf_local[v__8.s7])) + ((convert_float8(vload8(0, layer_norm148 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_h_to_4h_weight4 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))))); + NT_matmul_intermediate_pad_rf_local[v__8.s0] = v__9.s0; + NT_matmul_intermediate_pad_rf_local[v__8.s1] = v__9.s1; + NT_matmul_intermediate_pad_rf_local[v__8.s2] = v__9.s2; + NT_matmul_intermediate_pad_rf_local[v__8.s3] = v__9.s3; + NT_matmul_intermediate_pad_rf_local[v__8.s4] = v__9.s4; + NT_matmul_intermediate_pad_rf_local[v__8.s5] = v__9.s5; + NT_matmul_intermediate_pad_rf_local[v__8.s6] = v__9.s6; + NT_matmul_intermediate_pad_rf_local[v__8.s7] = v__9.s7; + } + if ((((convert_int(get_group_id(1))) * 4) + 2) < batch_size) { + int8 v__10 = (int8)((4)+(8*0), (4)+(8*1), (4)+(8*2), (4)+(8*3), (4)+(8*4), (4)+(8*5), (4)+(8*6), (4)+(8*7)); + float8 v__11 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__10.s0],NT_matmul_intermediate_pad_rf_local[v__10.s1],NT_matmul_intermediate_pad_rf_local[v__10.s2],NT_matmul_intermediate_pad_rf_local[v__10.s3],NT_matmul_intermediate_pad_rf_local[v__10.s4],NT_matmul_intermediate_pad_rf_local[v__10.s5],NT_matmul_intermediate_pad_rf_local[v__10.s6],NT_matmul_intermediate_pad_rf_local[v__10.s7])) + ((convert_float8(vload8(0, layer_norm148 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 4096)))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_h_to_4h_weight4 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))))); + NT_matmul_intermediate_pad_rf_local[v__10.s0] = v__11.s0; + NT_matmul_intermediate_pad_rf_local[v__10.s1] = v__11.s1; + NT_matmul_intermediate_pad_rf_local[v__10.s2] = v__11.s2; + NT_matmul_intermediate_pad_rf_local[v__10.s3] = v__11.s3; + NT_matmul_intermediate_pad_rf_local[v__10.s4] = v__11.s4; + NT_matmul_intermediate_pad_rf_local[v__10.s5] = v__11.s5; + NT_matmul_intermediate_pad_rf_local[v__10.s6] = v__11.s6; + NT_matmul_intermediate_pad_rf_local[v__10.s7] = v__11.s7; + int8 v__12 = (int8)((5)+(8*0), (5)+(8*1), (5)+(8*2), (5)+(8*3), (5)+(8*4), (5)+(8*5), (5)+(8*6), (5)+(8*7)); + float8 v__13 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__12.s0],NT_matmul_intermediate_pad_rf_local[v__12.s1],NT_matmul_intermediate_pad_rf_local[v__12.s2],NT_matmul_intermediate_pad_rf_local[v__12.s3],NT_matmul_intermediate_pad_rf_local[v__12.s4],NT_matmul_intermediate_pad_rf_local[v__12.s5],NT_matmul_intermediate_pad_rf_local[v__12.s6],NT_matmul_intermediate_pad_rf_local[v__12.s7])) + ((convert_float8(vload8(0, layer_norm148 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 4096)))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_h_to_4h_weight4 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))))); + NT_matmul_intermediate_pad_rf_local[v__12.s0] = v__13.s0; + NT_matmul_intermediate_pad_rf_local[v__12.s1] = v__13.s1; + NT_matmul_intermediate_pad_rf_local[v__12.s2] = v__13.s2; + NT_matmul_intermediate_pad_rf_local[v__12.s3] = v__13.s3; + NT_matmul_intermediate_pad_rf_local[v__12.s4] = v__13.s4; + NT_matmul_intermediate_pad_rf_local[v__12.s5] = v__13.s5; + NT_matmul_intermediate_pad_rf_local[v__12.s6] = v__13.s6; + NT_matmul_intermediate_pad_rf_local[v__12.s7] = v__13.s7; + } + if ((((convert_int(get_group_id(1))) * 4) + 3) < batch_size) { + int8 v__14 = (int8)((6)+(8*0), (6)+(8*1), (6)+(8*2), (6)+(8*3), (6)+(8*4), (6)+(8*5), (6)+(8*6), (6)+(8*7)); + float8 v__15 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__14.s0],NT_matmul_intermediate_pad_rf_local[v__14.s1],NT_matmul_intermediate_pad_rf_local[v__14.s2],NT_matmul_intermediate_pad_rf_local[v__14.s3],NT_matmul_intermediate_pad_rf_local[v__14.s4],NT_matmul_intermediate_pad_rf_local[v__14.s5],NT_matmul_intermediate_pad_rf_local[v__14.s6],NT_matmul_intermediate_pad_rf_local[v__14.s7])) + ((convert_float8(vload8(0, layer_norm148 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 6144)))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_h_to_4h_weight4 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))))); + NT_matmul_intermediate_pad_rf_local[v__14.s0] = v__15.s0; + NT_matmul_intermediate_pad_rf_local[v__14.s1] = v__15.s1; + NT_matmul_intermediate_pad_rf_local[v__14.s2] = v__15.s2; + NT_matmul_intermediate_pad_rf_local[v__14.s3] = v__15.s3; + NT_matmul_intermediate_pad_rf_local[v__14.s4] = v__15.s4; + NT_matmul_intermediate_pad_rf_local[v__14.s5] = v__15.s5; + NT_matmul_intermediate_pad_rf_local[v__14.s6] = v__15.s6; + NT_matmul_intermediate_pad_rf_local[v__14.s7] = v__15.s7; + int8 v__16 = (int8)((7)+(8*0), (7)+(8*1), (7)+(8*2), (7)+(8*3), (7)+(8*4), (7)+(8*5), (7)+(8*6), (7)+(8*7)); + float8 v__17 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__16.s0],NT_matmul_intermediate_pad_rf_local[v__16.s1],NT_matmul_intermediate_pad_rf_local[v__16.s2],NT_matmul_intermediate_pad_rf_local[v__16.s3],NT_matmul_intermediate_pad_rf_local[v__16.s4],NT_matmul_intermediate_pad_rf_local[v__16.s5],NT_matmul_intermediate_pad_rf_local[v__16.s6],NT_matmul_intermediate_pad_rf_local[v__16.s7])) + ((convert_float8(vload8(0, layer_norm148 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 6144)))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_h_to_4h_weight4 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))))); + NT_matmul_intermediate_pad_rf_local[v__16.s0] = v__17.s0; + NT_matmul_intermediate_pad_rf_local[v__16.s1] = v__17.s1; + NT_matmul_intermediate_pad_rf_local[v__16.s2] = v__17.s2; + NT_matmul_intermediate_pad_rf_local[v__16.s3] = v__17.s3; + NT_matmul_intermediate_pad_rf_local[v__16.s4] = v__17.s4; + NT_matmul_intermediate_pad_rf_local[v__16.s5] = v__17.s5; + NT_matmul_intermediate_pad_rf_local[v__16.s6] = v__17.s6; + NT_matmul_intermediate_pad_rf_local[v__16.s7] = v__17.s7; + } + } + for (int ax3_fused_2_0 = 0; ax3_fused_2_0 < 1; ++ax3_fused_2_0) { + for (int ax2 = 0; ax2 < 4; ++ax2) { + vstore2(((float2)(0.000000e+00f, 0.000000e+00f)), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + (ax2 * 2))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 8))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 16))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 24))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 32))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 40))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 48))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 56))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + } + } + for (int ax2_fused_2 = 0; ax2_fused_2 < 2; ++ax2_fused_2) { + for (int ax1 = 0; ax1 < 4; ++ax1) { + barrier(CLK_LOCAL_MEM_FENCE); + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_intermediate_pad_rf_local_1[((ax1 * 2) + ax2_fused_2)]; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 16) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 8) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 4) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 2) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + NT_matmul_intermediate_pad_local[((ax1 * 2) + ax2_fused_2)] = ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) * 32)]; + } + } + for (int ax0 = 0; ax0 < 4; ++ax0) { + for (int ax1_fused_2 = 0; ax1_fused_2 < 2; ++ax1_fused_2) { + if (((bool)(((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)(((bool)(((convert_int(get_group_id(1))) - ((batch_size + 3) >> 2)) < 0)) || ((bool)((convert_int(get_group_id(1))) == 0)))))) && ((bool)((((convert_int(get_group_id(1))) * 4) + ax0) < batch_size))) { + compute[((((((convert_int(get_group_id(1))) * 32768) + (ax0 * 8192)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] = (convert_half(((NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]) * 7.071068e-01f)) * 5.000000e-01f))))); + } + } + } +} + +// Function: gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2 +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len) { + half8 NT_matmul_intermediate_reindex_pad_local[2]; + for (int var = 0; var < 1; ++var) { + NT_matmul_intermediate_reindex_pad_local[0] = ((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)); + NT_matmul_intermediate_reindex_pad_local[1] = ((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)); + for (int ax3_0 = 0; ax3_0 < 128; ++ax3_0) { + for (int ax3_1 = 0; ax3_1 < 16; ++ax3_1) { + half condval; + if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < seq_len)) { + condval = layer_norm98[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; + } else { + condval = (half)0.000000e+00f; + } + int8 v_ = (int8)(((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*0), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*1), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*2), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*3), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*4), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*5), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*6), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*7)); + NT_matmul_intermediate_reindex_pad_local[0] = (NT_matmul_intermediate_reindex_pad_local[0] + (((half8)(condval, condval, condval, condval, condval, condval, condval, condval)) * ((half8)(gpt_neox_layers_0_attention_query_key_value_weight3[v_.s0],gpt_neox_layers_0_attention_query_key_value_weight3[v_.s1],gpt_neox_layers_0_attention_query_key_value_weight3[v_.s2],gpt_neox_layers_0_attention_query_key_value_weight3[v_.s3],gpt_neox_layers_0_attention_query_key_value_weight3[v_.s4],gpt_neox_layers_0_attention_query_key_value_weight3[v_.s5],gpt_neox_layers_0_attention_query_key_value_weight3[v_.s6],gpt_neox_layers_0_attention_query_key_value_weight3[v_.s7])))); + half condval_1; + if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < seq_len)) { + condval_1 = layer_norm98[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; + } else { + condval_1 = (half)0.000000e+00f; + } + int8 v__1 = (int8)(((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*0), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*1), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*2), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*3), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*4), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*5), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*6), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*7)); + NT_matmul_intermediate_reindex_pad_local[1] = (NT_matmul_intermediate_reindex_pad_local[1] + (((half8)(condval_1, condval_1, condval_1, condval_1, condval_1, condval_1, condval_1, condval_1)) * ((half8)(gpt_neox_layers_0_attention_query_key_value_weight3[v__1.s0],gpt_neox_layers_0_attention_query_key_value_weight3[v__1.s1],gpt_neox_layers_0_attention_query_key_value_weight3[v__1.s2],gpt_neox_layers_0_attention_query_key_value_weight3[v__1.s3],gpt_neox_layers_0_attention_query_key_value_weight3[v__1.s4],gpt_neox_layers_0_attention_query_key_value_weight3[v__1.s5],gpt_neox_layers_0_attention_query_key_value_weight3[v__1.s6],gpt_neox_layers_0_attention_query_key_value_weight3[v__1.s7])))); + } + } + if ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < seq_len) { + vstore8((NT_matmul_intermediate_reindex_pad_local[0] + vload8(0, gpt_neox_layers_0_attention_query_key_value_bias3 + (((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)))), 0, T_add + (((((convert_int(get_group_id(1))) * 49152) + ((convert_int(get_local_id(1))) * 12288)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8))); + } + if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < seq_len) { + vstore8((NT_matmul_intermediate_reindex_pad_local[1] + vload8(0, gpt_neox_layers_0_attention_query_key_value_bias3 + (((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)))), 0, T_add + ((((((convert_int(get_group_id(1))) * 49152) + ((convert_int(get_local_id(1))) * 12288)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 6144)); + } + } +} + +// Function: gpt_neox_q0f16_apply_penalty_inplace_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size) { + if ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) < num_token) { + logits[((seq_ids[pos2seq_id[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]] * vocab_size) + token_ids[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))])] = (logits[((seq_ids[pos2seq_id[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]] * vocab_size) + token_ids[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))])] - (penalties[(pos2seq_id[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] * 3)] + ((convert_float(token_cnt[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))])) * penalties[((pos2seq_id[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] * 3) + 1)]))); + float condval; + if ((logits[((seq_ids[pos2seq_id[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]] * vocab_size) + token_ids[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))])] < 0.000000e+00f)) { + condval = (logits[((seq_ids[pos2seq_id[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]] * vocab_size) + token_ids[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))])] * penalties[((pos2seq_id[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] * 3) + 2)]); + } else { + condval = (logits[((seq_ids[pos2seq_id[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]] * vocab_size) + token_ids[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))])] / penalties[((pos2seq_id[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] * 3) + 2)]); + } + logits[((seq_ids[pos2seq_id[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]] * vocab_size) + token_ids[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))])] = condval; + } +} + +// Function: gpt_neox_q0f16_softmax_with_chunked_sum_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size) { + float in_thread_temp_max_shared[1]; + __local float red_buf0[256]; + __local float temp_max_shared[1]; + float in_thread_temp_sum_shared[1]; + __local float red_buf0_1[256]; + __local float temp_sum_shared[1]; + in_thread_temp_max_shared[0] = -3.402823e+38f; + for (int ax0_0 = 0; ax0_0 < ((num_chunks + 31) >> 5); ++ax0_0) { + if (((ax0_0 * 32) + (convert_int(get_local_id(0)))) < num_chunks) { + in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)chunked_max[(((ax0_0 * 32) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * num_chunks)) + (convert_int(get_local_id(0))))]); + } + } + barrier(CLK_LOCAL_MEM_FENCE); + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] = in_thread_temp_max_shared[0]; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 16) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] = max((float)((volatile __local float*)red_buf0)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))], (float)((volatile __local float*)red_buf0)[((((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0)))) + 16)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 8) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] = max((float)((volatile __local float*)red_buf0)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))], (float)((volatile __local float*)red_buf0)[((((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0)))) + 8)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 4) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] = max((float)((volatile __local float*)red_buf0)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))], (float)((volatile __local float*)red_buf0)[((((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0)))) + 4)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 2) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] = max((float)((volatile __local float*)red_buf0)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))], (float)((volatile __local float*)red_buf0)[((((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0)))) + 2)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 1) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] = max((float)((volatile __local float*)red_buf0)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))], (float)((volatile __local float*)red_buf0)[((((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0)))) + 1)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) == 0) { + temp_max_shared[0] = ((volatile __local float*)red_buf0)[((convert_int(get_local_id(1))) * 32)]; + } + in_thread_temp_sum_shared[0] = 0.000000e+00f; + barrier(CLK_LOCAL_MEM_FENCE); + for (int ax0_0_1 = 0; ax0_0_1 < ((num_chunks + 31) >> 5); ++ax0_0_1) { + if (((ax0_0_1 * 32) + (convert_int(get_local_id(0)))) < num_chunks) { + float condval; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval = exp(((chunked_sum[(((ax0_0_1 * 32) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * num_chunks)) + (convert_int(get_local_id(0))))] + chunked_max[(((ax0_0_1 * 32) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * num_chunks)) + (convert_int(get_local_id(0))))]) - temp_max_shared[0])); + } else { + condval = ((convert_float((chunked_max[(((ax0_0_1 * 32) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * num_chunks)) + (convert_int(get_local_id(0))))] == temp_max_shared[0]))) * chunked_sum[(((ax0_0_1 * 32) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * num_chunks)) + (convert_int(get_local_id(0))))]); + } + in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval); + } + } + barrier(CLK_LOCAL_MEM_FENCE); + ((volatile __local float*)red_buf0_1)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] = in_thread_temp_sum_shared[0]; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 16) { + ((volatile __local float*)red_buf0_1)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] = (((volatile __local float*)red_buf0_1)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] + ((volatile __local float*)red_buf0_1)[((((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0)))) + 16)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 8) { + ((volatile __local float*)red_buf0_1)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] = (((volatile __local float*)red_buf0_1)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] + ((volatile __local float*)red_buf0_1)[((((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0)))) + 8)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 4) { + ((volatile __local float*)red_buf0_1)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] = (((volatile __local float*)red_buf0_1)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] + ((volatile __local float*)red_buf0_1)[((((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0)))) + 4)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 2) { + ((volatile __local float*)red_buf0_1)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] = (((volatile __local float*)red_buf0_1)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] + ((volatile __local float*)red_buf0_1)[((((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0)))) + 2)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 1) { + ((volatile __local float*)red_buf0_1)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] = (((volatile __local float*)red_buf0_1)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] + ((volatile __local float*)red_buf0_1)[((((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0)))) + 1)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) == 0) { + temp_sum_shared[0] = ((volatile __local float*)red_buf0_1)[((convert_int(get_local_id(1))) * 32)]; + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + (convert_int(get_local_id(0)))) < vocab_size) { + float condval_1; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_1 = exp(((A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0))))] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]) - (log(temp_sum_shared[0]) + temp_max_shared[0]))); + } else { + condval_1 = ((convert_float((A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0))))] == temp_max_shared[0]))) / temp_sum_shared[0]); + } + softmax[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0))))] = condval_1; + } + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + (convert_int(get_local_id(0)))) + 256) < vocab_size) { + float condval_2; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_2 = exp(((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 256)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]) - (log(temp_sum_shared[0]) + temp_max_shared[0]))); + } else { + condval_2 = ((convert_float((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 256)] == temp_max_shared[0]))) / temp_sum_shared[0]); + } + softmax[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 256)] = condval_2; + } + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + (convert_int(get_local_id(0)))) + 512) < vocab_size) { + float condval_3; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_3 = exp(((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 512)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]) - (log(temp_sum_shared[0]) + temp_max_shared[0]))); + } else { + condval_3 = ((convert_float((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 512)] == temp_max_shared[0]))) / temp_sum_shared[0]); + } + softmax[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 512)] = condval_3; + } + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + (convert_int(get_local_id(0)))) + 768) < vocab_size) { + float condval_4; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_4 = exp(((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 768)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]) - (log(temp_sum_shared[0]) + temp_max_shared[0]))); + } else { + condval_4 = ((convert_float((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 768)] == temp_max_shared[0]))) / temp_sum_shared[0]); + } + softmax[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 768)] = condval_4; + } + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + (convert_int(get_local_id(0)))) + 1024) < vocab_size) { + float condval_5; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_5 = exp(((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1024)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]) - (log(temp_sum_shared[0]) + temp_max_shared[0]))); + } else { + condval_5 = ((convert_float((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1024)] == temp_max_shared[0]))) / temp_sum_shared[0]); + } + softmax[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1024)] = condval_5; + } + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + (convert_int(get_local_id(0)))) + 1280) < vocab_size) { + float condval_6; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_6 = exp(((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1280)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]) - (log(temp_sum_shared[0]) + temp_max_shared[0]))); + } else { + condval_6 = ((convert_float((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1280)] == temp_max_shared[0]))) / temp_sum_shared[0]); + } + softmax[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1280)] = condval_6; + } + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + (convert_int(get_local_id(0)))) + 1536) < vocab_size) { + float condval_7; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_7 = exp(((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1536)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]) - (log(temp_sum_shared[0]) + temp_max_shared[0]))); + } else { + condval_7 = ((convert_float((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1536)] == temp_max_shared[0]))) / temp_sum_shared[0]); + } + softmax[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1536)] = condval_7; + } + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + (convert_int(get_local_id(0)))) + 1792) < vocab_size) { + float condval_8; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_8 = exp(((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1792)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]) - (log(temp_sum_shared[0]) + temp_max_shared[0]))); + } else { + condval_8 = ((convert_float((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1792)] == temp_max_shared[0]))) / temp_sum_shared[0]); + } + softmax[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1792)] = condval_8; + } + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + (convert_int(get_local_id(0)))) + 2048) < vocab_size) { + float condval_9; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_9 = exp(((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2048)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]) - (log(temp_sum_shared[0]) + temp_max_shared[0]))); + } else { + condval_9 = ((convert_float((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2048)] == temp_max_shared[0]))) / temp_sum_shared[0]); + } + softmax[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2048)] = condval_9; + } + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + (convert_int(get_local_id(0)))) + 2304) < vocab_size) { + float condval_10; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_10 = exp(((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2304)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]) - (log(temp_sum_shared[0]) + temp_max_shared[0]))); + } else { + condval_10 = ((convert_float((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2304)] == temp_max_shared[0]))) / temp_sum_shared[0]); + } + softmax[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2304)] = condval_10; + } + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + (convert_int(get_local_id(0)))) + 2560) < vocab_size) { + float condval_11; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_11 = exp(((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2560)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]) - (log(temp_sum_shared[0]) + temp_max_shared[0]))); + } else { + condval_11 = ((convert_float((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2560)] == temp_max_shared[0]))) / temp_sum_shared[0]); + } + softmax[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2560)] = condval_11; + } + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + (convert_int(get_local_id(0)))) + 2816) < vocab_size) { + float condval_12; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_12 = exp(((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2816)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]) - (log(temp_sum_shared[0]) + temp_max_shared[0]))); + } else { + condval_12 = ((convert_float((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2816)] == temp_max_shared[0]))) / temp_sum_shared[0]); + } + softmax[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2816)] = condval_12; + } + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + (convert_int(get_local_id(0)))) + 3072) < vocab_size) { + float condval_13; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_13 = exp(((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3072)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]) - (log(temp_sum_shared[0]) + temp_max_shared[0]))); + } else { + condval_13 = ((convert_float((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3072)] == temp_max_shared[0]))) / temp_sum_shared[0]); + } + softmax[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3072)] = condval_13; + } + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + (convert_int(get_local_id(0)))) + 3328) < vocab_size) { + float condval_14; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_14 = exp(((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3328)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]) - (log(temp_sum_shared[0]) + temp_max_shared[0]))); + } else { + condval_14 = ((convert_float((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3328)] == temp_max_shared[0]))) / temp_sum_shared[0]); + } + softmax[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3328)] = condval_14; + } + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + (convert_int(get_local_id(0)))) + 3584) < vocab_size) { + float condval_15; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_15 = exp(((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3584)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]) - (log(temp_sum_shared[0]) + temp_max_shared[0]))); + } else { + condval_15 = ((convert_float((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3584)] == temp_max_shared[0]))) / temp_sum_shared[0]); + } + softmax[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3584)] = condval_15; + } + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + (convert_int(get_local_id(0)))) + 3840) < vocab_size) { + float condval_16; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_16 = exp(((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3840)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]) - (log(temp_sum_shared[0]) + temp_max_shared[0]))); + } else { + condval_16 = ((convert_float((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3840)] == temp_max_shared[0]))) / temp_sum_shared[0]); + } + softmax[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3840)] = condval_16; + } +} + +// Function: gpt_neox_q0f16_batch_tree_attn_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size) { + int tile_id[1]; + int batch_idx[1]; + int batch_rows[1]; + int batch_tiles[1]; + int kv_chunk_len[1]; + __local float m_smem[32]; + __local float d_smem[32]; + float O_local[32]; + __local half Q_smem[4096]; + __local half K_smem[4096]; + __local half V_smem[4096]; + float S_local[8]; + __local float S_smem[1024]; + float m_prev[1]; + float m_new[1]; + float d_new[1]; + __local float m_prev_smem[32]; + tile_id[0] = (convert_int(get_group_id(0))); + batch_idx[0] = 0; + batch_rows[0] = (q_indptr[(q_indptr_elem_offset + 1)] - q_indptr[q_indptr_elem_offset]); + batch_tiles[0] = ((batch_rows[0] + 31) >> 5); + while (1) { + if (!(((batch_idx[0] < batch_size)))) { break; } + while (1) { + if (!((((bool)(batch_tiles[0] <= tile_id[0])) && ((bool)(batch_idx[0] < batch_size))))) { break; } + tile_id[0] = (tile_id[0] - batch_tiles[0]); + batch_idx[0] = (batch_idx[0] + 1); + if (batch_idx[0] < batch_size) { + int b_idx = batch_idx[0]; + batch_rows[0] = (q_indptr[((b_idx + q_indptr_elem_offset) + 1)] - q_indptr[(b_idx + q_indptr_elem_offset)]); + batch_tiles[0] = ((batch_rows[0] + 31) >> 5); + } + } + if ((batch_idx[0] < batch_size)) { + int b_idx_1 = batch_idx[0]; + int LH_start = (tile_id[0] * 32); + int q_indptr_val = q_indptr[(b_idx_1 + q_indptr_elem_offset)]; + kv_chunk_len[0] = (kv_indptr[((b_idx_1 + kv_indptr_elem_offset) + 1)] - kv_indptr[(b_idx_1 + kv_indptr_elem_offset)]); + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + m_smem[(convert_int(get_local_id(0)))] = -5.000000e+04f; + d_smem[(convert_int(get_local_id(0)))] = 1.000000e+00f; + } + for (int li_1 = 0; li_1 < 4; ++li_1) { + for (int lj_1 = 0; lj_1 < 8; ++lj_1) { + O_local[((li_1 * 8) + lj_1)] = 0.000000e+00f; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + for (int li_lj_fused_0 = 0; li_lj_fused_0 < 8; ++li_lj_fused_0) { + if (((((li_lj_fused_0 * 4) + q_indptr_val) + LH_start) + (convert_int(get_local_id(1)))) < q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)]) { + half4 condval; + if ((rotary_mode == 1)) { + float4 freq = (((float4)(((convert_float(q_rope_position[(((((li_lj_fused_0 * 4) + (convert_int(get_local_id(1)))) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[(((((li_lj_fused_0 * 4) + (convert_int(get_local_id(1)))) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[(((((li_lj_fused_0 * 4) + (convert_int(get_local_id(1)))) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[(((((li_lj_fused_0 * 4) + (convert_int(get_local_id(1)))) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale))) / pow(((float4)(rope_theta, rope_theta, rope_theta, rope_theta)), ((convert_float4(((int4)(((((convert_int(get_local_id(0))) & 15) * 8))+(2*0), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*1), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*2), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*3))))) * ((float4)(7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f))))); + half4 condval_1; + if (((convert_int(get_local_id(0))) < 16)) { + condval_1 = (vload4(0, q + (((((((li_lj_fused_0 * 8192) + ((convert_int(get_local_id(1))) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + 64)) * ((half4)((half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f))); + } else { + condval_1 = vload4(0, q + (((((((li_lj_fused_0 * 8192) + ((convert_int(get_local_id(1))) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) - 64)); + } + condval = (convert_half4(((cos(freq) * (convert_float4(vload4(0, q + ((((((li_lj_fused_0 * 8192) + ((convert_int(get_local_id(1))) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)))))) + (sin(freq) * (convert_float4(condval_1)))))); + } else { + condval = vload4(0, q + ((((((li_lj_fused_0 * 8192) + ((convert_int(get_local_id(1))) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); + } + vstore4(condval, 0, Q_smem + (((li_lj_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); + } else { + vstore4(((half4)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, Q_smem + (((li_lj_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); + } + } + barrier(CLK_LOCAL_MEM_FENCE); + for (int iterator = 0; iterator < ((kv_chunk_len[0] + 31) >> 5); ++iterator) { + int L_kv_base = kv_indptr[(b_idx_1 + kv_indptr_elem_offset)]; + barrier(CLK_LOCAL_MEM_FENCE); + for (int lz_ly_fused_0 = 0; lz_ly_fused_0 < 8; ++lz_ly_fused_0) { + if ((((iterator * 32) + (lz_ly_fused_0 * 4)) + (convert_int(get_local_id(1)))) < kv_chunk_len[0]) { + half4 condval_2; + if ((rotary_mode == 1)) { + float4 freq_1 = (((float4)(((convert_float(q_rope_position[(((((iterator * 32) + (lz_ly_fused_0 * 4)) + (convert_int(get_local_id(1)))) + L_kv_base) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[(((((iterator * 32) + (lz_ly_fused_0 * 4)) + (convert_int(get_local_id(1)))) + L_kv_base) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[(((((iterator * 32) + (lz_ly_fused_0 * 4)) + (convert_int(get_local_id(1)))) + L_kv_base) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[(((((iterator * 32) + (lz_ly_fused_0 * 4)) + (convert_int(get_local_id(1)))) + L_kv_base) + q_rope_position_elem_offset)])) * rope_scale))) / pow(((float4)(rope_theta, rope_theta, rope_theta, rope_theta)), ((convert_float4(((int4)(((((convert_int(get_local_id(0))) & 15) * 8))+(2*0), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*1), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*2), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*3))))) * ((float4)(7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f))))); + half4 condval_3; + if (((convert_int(get_local_id(0))) < 16)) { + condval_3 = (vload4(0, k + (((((((iterator * 65536) + (lz_ly_fused_0 * 8192)) + ((convert_int(get_local_id(1))) * 2048)) + (L_kv_base * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + 64)) * ((half4)((half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f))); + } else { + condval_3 = vload4(0, k + (((((((iterator * 65536) + (lz_ly_fused_0 * 8192)) + ((convert_int(get_local_id(1))) * 2048)) + (L_kv_base * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) - 64)); + } + condval_2 = (convert_half4(((cos(freq_1) * (convert_float4(vload4(0, k + ((((((iterator * 65536) + (lz_ly_fused_0 * 8192)) + ((convert_int(get_local_id(1))) * 2048)) + (L_kv_base * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)))))) + (sin(freq_1) * (convert_float4(condval_3)))))); + } else { + condval_2 = vload4(0, k + ((((((iterator * 65536) + (lz_ly_fused_0 * 8192)) + ((convert_int(get_local_id(1))) * 2048)) + (L_kv_base * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); + } + vstore4(condval_2, 0, K_smem + (((lz_ly_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); + vstore4(vload4(0, v + ((((((iterator * 65536) + (lz_ly_fused_0 * 8192)) + ((convert_int(get_local_id(1))) * 2048)) + (L_kv_base * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))), 0, V_smem + (((lz_ly_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); + } else { + vstore4(((half4)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, K_smem + (((lz_ly_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); + vstore4(((half4)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, V_smem + (((lz_ly_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); + } + } + barrier(CLK_LOCAL_MEM_FENCE); + for (int li_1_init = 0; li_1_init < 2; ++li_1_init) { + for (int lj_1_init = 0; lj_1_init < 4; ++lj_1_init) { + S_local[((li_1_init * 4) + lj_1_init)] = 0.000000e+00f; + } + } + for (int lk_0 = 0; lk_0 < 16; ++lk_0) { + for (int li_1_1 = 0; li_1_1 < 2; ++li_1_1) { + for (int lj_1_1 = 0; lj_1_1 < 4; ++lj_1_1) { + for (int lk_1 = 0; lk_1 < 8; ++lk_1) { + S_local[((li_1_1 * 4) + lj_1_1)] = (S_local[((li_1_1 * 4) + lj_1_1)] + ((((convert_float(Q_smem[((((((convert_int(get_local_id(1))) * 1024) + (((convert_int(get_local_id(0))) >> 3) * 256)) + (li_1_1 * 128)) + (lk_0 * 8)) + lk_1)])) * (convert_float(K_smem[((((((convert_int(get_local_id(0))) & 7) * 512) + (lj_1_1 * 128)) + (lk_0 * 8)) + lk_1)]))) * attn_score_scaling_factor) * 1.275174e-01f)); + } + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + for (int li_1_2 = 0; li_1_2 < 2; ++li_1_2) { + for (int lj_1_2 = 0; lj_1_2 < 4; ++lj_1_2) { + S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 3) * 64)) + (li_1_2 * 32)) + (((convert_int(get_local_id(0))) & 7) * 4)) + lj_1_2)] = S_local[((li_1_2 * 4) + lj_1_2)]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + m_prev[0] = m_smem[(convert_int(get_local_id(0)))]; + m_new[0] = m_smem[(convert_int(get_local_id(0)))]; + for (int j = 0; j < 32; ++j) { + int cse_var_4 = ((iterator * 32) + j); + int cse_var_3 = ((iterator * 64) + (j * 2)); + if (((bool)(cse_var_4 < kv_chunk_len[0])) && ((bool)(((bool)(cse_var_4 < ((kv_chunk_len[0] + mn_indptr[(b_idx_1 + mn_indptr_elem_offset)]) - mn_indptr[((b_idx_1 + mn_indptr_elem_offset) + 1)]))) || ((bool)(((bool)(mask[((((mn_indptr[((b_idx_1 + mn_indptr_elem_offset) + 1)] * 2) + cse_var_3) + mask_elem_offset) - (kv_chunk_len[0] * 2))] <= mask[(((((((convert_int(get_local_id(0))) * 2) + (LH_start * 2)) + (mn_indptr[((b_idx_1 + mn_indptr_elem_offset) + 1)] * 2)) + (q_indptr[(b_idx_1 + q_indptr_elem_offset)] * 2)) + mask_elem_offset) - (q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)] * 2))])) && ((bool)(mask[(((((((convert_int(get_local_id(0))) * 2) + (LH_start * 2)) + (mn_indptr[((b_idx_1 + mn_indptr_elem_offset) + 1)] * 2)) + (q_indptr[(b_idx_1 + q_indptr_elem_offset)] * 2)) + mask_elem_offset) - (q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)] * 2))] < mask[(((((mn_indptr[((b_idx_1 + mn_indptr_elem_offset) + 1)] * 2) + cse_var_3) + mask_elem_offset) + 1) - (kv_chunk_len[0] * 2))]))))))) { + m_new[0] = max((float)m_new[0], (float)S_smem[((((convert_int(get_local_id(1))) * 1024) + ((convert_int(get_local_id(0))) * 32)) + j)]); + } + } + d_new[0] = (d_smem[(convert_int(get_local_id(0)))] * exp2((m_prev[0] - m_new[0]))); + } + for (int j_1 = 0; j_1 < 32; ++j_1) { + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + int cse_var_6 = ((iterator * 32) + j_1); + int cse_var_5 = ((iterator * 64) + (j_1 * 2)); + if (((bool)(cse_var_6 < kv_chunk_len[0])) && ((bool)(((bool)(cse_var_6 < ((kv_chunk_len[0] + mn_indptr[(b_idx_1 + mn_indptr_elem_offset)]) - mn_indptr[((b_idx_1 + mn_indptr_elem_offset) + 1)]))) || ((bool)(((bool)(mask[((((mn_indptr[((b_idx_1 + mn_indptr_elem_offset) + 1)] * 2) + cse_var_5) + mask_elem_offset) - (kv_chunk_len[0] * 2))] <= mask[(((((((convert_int(get_local_id(0))) * 2) + (LH_start * 2)) + (mn_indptr[((b_idx_1 + mn_indptr_elem_offset) + 1)] * 2)) + (q_indptr[(b_idx_1 + q_indptr_elem_offset)] * 2)) + mask_elem_offset) - (q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)] * 2))])) && ((bool)(mask[(((((((convert_int(get_local_id(0))) * 2) + (LH_start * 2)) + (mn_indptr[((b_idx_1 + mn_indptr_elem_offset) + 1)] * 2)) + (q_indptr[(b_idx_1 + q_indptr_elem_offset)] * 2)) + mask_elem_offset) - (q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)] * 2))] < mask[(((((mn_indptr[((b_idx_1 + mn_indptr_elem_offset) + 1)] * 2) + cse_var_5) + mask_elem_offset) + 1) - (kv_chunk_len[0] * 2))]))))))) { + S_smem[((((convert_int(get_local_id(1))) * 1024) + ((convert_int(get_local_id(0))) * 32)) + j_1)] = exp2((S_smem[((((convert_int(get_local_id(1))) * 1024) + ((convert_int(get_local_id(0))) * 32)) + j_1)] - m_new[0])); + } else { + S_smem[(((convert_int(get_local_id(0))) * 32) + j_1)] = exp2((-5.000000e+04f - m_new[0])); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + for (int j_2 = 0; j_2 < 32; ++j_2) { + d_new[0] = (d_new[0] + S_smem[(((convert_int(get_local_id(0))) * 32) + j_2)]); + } + m_smem[(convert_int(get_local_id(0)))] = m_new[0]; + d_smem[(convert_int(get_local_id(0)))] = d_new[0]; + m_prev_smem[(convert_int(get_local_id(0)))] = m_prev[0]; + } + barrier(CLK_LOCAL_MEM_FENCE); + for (int li_1_init_1 = 0; li_1_init_1 < 4; ++li_1_init_1) { + for (int lj_1_init_1 = 0; lj_1_init_1 < 8; ++lj_1_init_1) { + O_local[((li_1_init_1 * 8) + lj_1_init_1)] = (O_local[((li_1_init_1 * 8) + lj_1_init_1)] * exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_init_1)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_init_1)]))); + } + } + for (int lk_0_1 = 0; lk_0_1 < 4; ++lk_0_1) { + for (int lk_1_1 = 0; lk_1_1 < 8; ++lk_1_1) { + for (int li_1_3 = 0; li_1_3 < 4; ++li_1_3) { + for (int lj_1_3 = 0; lj_1_3 < 8; ++lj_1_3) { + O_local[((li_1_3 * 8) + lj_1_3)] = (O_local[((li_1_3 * 8) + lj_1_3)] + (S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (li_1_3 * 32)) + (lk_0_1 * 8)) + lk_1_1)] * (convert_float(V_smem[((((lk_0_1 * 1024) + (lk_1_1 * 128)) + (((convert_int(get_local_id(0))) & 15) * 8)) + lj_1_3)])))); + } + } + } + } + } + for (int li_1_4 = 0; li_1_4 < 4; ++li_1_4) { + for (int lj_1_4 = 0; lj_1_4 < 8; ++lj_1_4) { + int cur_L = ((((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + q_indptr[(b_idx_1 + q_indptr_elem_offset)]) + LH_start) + li_1_4); + if (cur_L < q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)]) { + output[((((cur_L * 2048) + ((convert_int(get_group_id(1))) * 128)) + (((convert_int(get_local_id(0))) & 15) * 8)) + lj_1_4)] = (convert_half((O_local[((li_1_4 * 8) + lj_1_4)] / d_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_4)]))); + } + } + } + if ((convert_int(get_local_id(1))) < 1) { + int cur_L_1 = (((((convert_int(get_local_id(1))) * 32) + q_indptr[(b_idx_1 + q_indptr_elem_offset)]) + LH_start) + (convert_int(get_local_id(0)))); + if (cur_L_1 < q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)]) { + lse[((cur_L_1 * 16) + (convert_int(get_group_id(1))))] = (m_smem[(convert_int(get_local_id(0)))] + log2(d_smem[(convert_int(get_local_id(0)))])); + } + } + tile_id[0] = (tile_id[0] + 16); + } + } +} + +// Function: gpt_neox_q0f16_batch_decode_paged_kv_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode) { + int kv_chunk_len[1]; + float st_m[1]; + float st_d[1]; + float4 O_local[1]; + half4 Q_local[1]; + float m_prev[1]; + __local float4 O_allreduce[256]; + __local float md_allreduce[16]; + __local half4 K_smem[256]; + __local half4 V_smem[256]; + float QK_local[4]; + float S_reduce_local[1]; + __local float red_buf0[256]; + float S_local[1]; + half4 V_local[1]; + float d_prev[1]; + float other_m[1]; + float other_d[1]; + float4 other_o[1]; + float exp_mprev[1]; + int cur_page_indptr_begin = page_table_indptr[((convert_int(get_group_id(0))) + page_indptr_elem_offset)]; + int cur_page_indptr_end = page_table_indptr[(((convert_int(get_group_id(0))) + page_indptr_elem_offset) + 1)]; + int condval; + if ((cur_page_indptr_begin != cur_page_indptr_end)) { + condval = ((((cur_page_indptr_end * 16) + length_info[((convert_int(get_group_id(0))) + length_info_elem_offset)]) - (cur_page_indptr_begin * 16)) - 16); + } else { + condval = 0; + } + kv_chunk_len[0] = condval; + st_m[0] = -5.000000e+04f; + st_d[0] = 1.000000e+00f; + O_local[0] = ((float4)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)); + half4 condval_1; + if ((rotary_mode == 1)) { + float4 freq = (((float4)(((convert_float(q_rope_position[((convert_int(get_group_id(0))) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[((convert_int(get_group_id(0))) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[((convert_int(get_group_id(0))) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[((convert_int(get_group_id(0))) + q_rope_position_elem_offset)])) * rope_scale))) / pow(((float4)(rope_theta, rope_theta, rope_theta, rope_theta)), ((convert_float4(((int4)(((((convert_int(get_local_id(0))) & 15) * 8))+(2*0), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*1), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*2), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*3))))) * ((float4)(7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f))))); + half4 condval_2; + if (((convert_int(get_local_id(0))) < 16)) { + condval_2 = (vload4(0, Q + (((((convert_int(get_group_id(0))) * 2048) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + 64)) * ((half4)((half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f))); + } else { + condval_2 = vload4(0, Q + (((((convert_int(get_group_id(0))) * 2048) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) - 64)); + } + condval_1 = (convert_half4(((cos(freq) * (convert_float4(vload4(0, Q + ((((convert_int(get_group_id(0))) * 2048) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)))))) + (sin(freq) * (convert_float4(condval_2)))))); + } else { + condval_1 = vload4(0, Q + ((((convert_int(get_group_id(0))) * 2048) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); + } + Q_local[0] = condval_1; + for (int iterator = 0; iterator < ((kv_chunk_len[0] + 7) >> 3); ++iterator) { + if (((iterator * 8) + (convert_int(get_local_id(2)))) < kv_chunk_len[0]) { + int page_no = page_table_values[(((iterator >> 1) + cur_page_indptr_begin) + page_values_elem_offset)]; + half4 condval_3; + if ((rotary_mode == 1)) { + float4 freq_1 = (((float4)(((convert_float((((iterator * 8) + k_rope_pos_offset[((convert_int(get_group_id(0))) + k_rope_pos_offset_elem_offset)]) + (convert_int(get_local_id(2)))))) * rope_scale), ((convert_float((((iterator * 8) + k_rope_pos_offset[((convert_int(get_group_id(0))) + k_rope_pos_offset_elem_offset)]) + (convert_int(get_local_id(2)))))) * rope_scale), ((convert_float((((iterator * 8) + k_rope_pos_offset[((convert_int(get_group_id(0))) + k_rope_pos_offset_elem_offset)]) + (convert_int(get_local_id(2)))))) * rope_scale), ((convert_float((((iterator * 8) + k_rope_pos_offset[((convert_int(get_group_id(0))) + k_rope_pos_offset_elem_offset)]) + (convert_int(get_local_id(2)))))) * rope_scale))) / pow(((float4)(rope_theta, rope_theta, rope_theta, rope_theta)), ((convert_float4(((int4)(((((convert_int(get_local_id(0))) & 15) * 8))+(2*0), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*1), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*2), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*3))))) * ((float4)(7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f))))); + int4 v_ = (int4)((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*0), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*1), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*2), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*3)); + half4 condval_4; + if (((convert_int(get_local_id(0))) < 16)) { + int4 v__1 = (int4)(((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 64))+(1*0), ((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 64))+(1*1), ((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 64))+(1*2), ((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 64))+(1*3)); + condval_4 = (((half4)(pages[v__1.s0],pages[v__1.s1],pages[v__1.s2],pages[v__1.s3])) * ((half4)((half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f))); + } else { + int4 v__2 = (int4)(((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) - 64))+(1*0), ((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) - 64))+(1*1), ((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) - 64))+(1*2), ((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) - 64))+(1*3)); + condval_4 = ((half4)(pages[v__2.s0],pages[v__2.s1],pages[v__2.s2],pages[v__2.s3])); + } + condval_3 = (convert_half4(((cos(freq_1) * (convert_float4(((half4)(pages[v_.s0],pages[v_.s1],pages[v_.s2],pages[v_.s3]))))) + (sin(freq_1) * (convert_float4(condval_4)))))); + } else { + int4 v__3 = (int4)((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*0), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*1), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*2), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*3)); + condval_3 = ((half4)(pages[v__3.s0],pages[v__3.s1],pages[v__3.s2],pages[v__3.s3])); + } + K_smem[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = condval_3; + int4 v__4 = (int4)(((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 32768))+(1*0), ((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 32768))+(1*1), ((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 32768))+(1*2), ((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 32768))+(1*3)); + V_smem[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = ((half4)(pages[v__4.s0],pages[v__4.s1],pages[v__4.s2],pages[v__4.s3])); + } else { + K_smem[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = ((half4)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)); + V_smem[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = ((half4)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)); + } + barrier(CLK_LOCAL_MEM_FENCE); + m_prev[0] = st_m[0]; + vstore4(((((convert_float4(Q_local[0])) * (convert_float4(K_smem[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))]))) * ((float4)(attn_score_scaling_factor, attn_score_scaling_factor, attn_score_scaling_factor, attn_score_scaling_factor))) * ((float4)(1.275174e-01f, 1.275174e-01f, 1.275174e-01f, 1.275174e-01f))), 0, QK_local + 0); + S_reduce_local[0] = 0.000000e+00f; + S_reduce_local[0] = (S_reduce_local[0] + QK_local[0]); + S_reduce_local[0] = (S_reduce_local[0] + QK_local[1]); + S_reduce_local[0] = (S_reduce_local[0] + QK_local[2]); + S_reduce_local[0] = (S_reduce_local[0] + QK_local[3]); + barrier(CLK_LOCAL_MEM_FENCE); + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = S_reduce_local[0]; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 16) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0)))) + 16)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 8) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0)))) + 8)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 4) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0)))) + 4)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 2) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0)))) + 2)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 1) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0)))) + 1)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + S_local[0] = -5.000000e+04f; + if (((iterator * 8) + (convert_int(get_local_id(2)))) < kv_chunk_len[0]) { + S_local[0] = ((volatile __local float*)red_buf0)[((convert_int(get_local_id(2))) * 32)]; + } + st_m[0] = max((float)st_m[0], (float)S_local[0]); + float o_scale = exp2((m_prev[0] - st_m[0])); + st_d[0] = (st_d[0] * o_scale); + S_local[0] = exp2((S_local[0] - st_m[0])); + st_d[0] = (st_d[0] + S_local[0]); + O_local[0] = (O_local[0] * ((float4)(o_scale, o_scale, o_scale, o_scale))); + V_local[0] = V_smem[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))]; + O_local[0] = (O_local[0] + ((convert_float4(V_local[0])) * ((float4)(S_local[0], S_local[0], S_local[0], S_local[0])))); + } + O_allreduce[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = O_local[0]; + md_allreduce[((convert_int(get_local_id(2))) * 2)] = st_m[0]; + md_allreduce[(((convert_int(get_local_id(2))) * 2) + 1)] = st_d[0]; + barrier(CLK_LOCAL_MEM_FENCE); + st_m[0] = -5.000000e+04f; + st_d[0] = 1.000000e+00f; + O_local[0] = ((float4)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)); + for (int j = 0; j < 8; ++j) { + m_prev[0] = st_m[0]; + d_prev[0] = st_d[0]; + other_m[0] = md_allreduce[(j * 2)]; + other_d[0] = md_allreduce[((j * 2) + 1)]; + other_o[0] = O_allreduce[((j * 32) + (convert_int(get_local_id(0))))]; + st_m[0] = max((float)st_m[0], (float)other_m[0]); + st_d[0] = ((d_prev[0] * exp2((m_prev[0] - st_m[0]))) + (other_d[0] * exp2((other_m[0] - st_m[0])))); + exp_mprev[0] = exp2((m_prev[0] - st_m[0])); + other_m[0] = exp2((other_m[0] - st_m[0])); + O_local[0] = ((O_local[0] * ((float4)(exp_mprev[0], exp_mprev[0], exp_mprev[0], exp_mprev[0]))) + (other_o[0] * ((float4)(other_m[0], other_m[0], other_m[0], other_m[0])))); + } + O_local[0] = (O_local[0] / ((float4)(st_d[0], st_d[0], st_d[0], st_d[0]))); + vstore4((convert_half4(O_local[0])), 0, output + ((((convert_int(get_group_id(0))) * 2048) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); + lse[(((convert_int(get_group_id(0))) * 16) + (convert_int(get_group_id(1))))] = (st_m[0] + log2(st_d[0])); +} + +// Function: gpt_neox_q0f16_reshape5_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len) { + T_reshape[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] = reshape192[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]; +} + +// Function: gpt_neox_q0f16_compact_kv_copy_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length) { + if (((convert_int(get_group_id(0))) - (batch_size * 8)) < 0) { + for (int i = 0; i < (copy_length_indptr[((((convert_int(get_group_id(0))) >> 3) + copy_length_indptr_elem_offset) + 1)] - copy_length_indptr[(((convert_int(get_group_id(0))) >> 3) + copy_length_indptr_elem_offset)]); ++i) { + int src_pos = copy_src_dst_pos[((copy_length_indptr[(((convert_int(get_group_id(0))) >> 3) + copy_length_indptr_elem_offset)] + i) + copy_src_dst_pos_elem_offset)]; + int dst_pos = copy_src_dst_pos[(((total_copy_length + copy_length_indptr[(((convert_int(get_group_id(0))) >> 3) + copy_length_indptr_elem_offset)]) + i) + copy_src_dst_pos_elem_offset)]; + pages[(((((((dst_pos >> 4) * 65536) + (((convert_int(get_group_id(0))) & 7) * 4096)) + (((convert_int(get_local_id(0))) >> 7) * 2048)) + ((dst_pos & 15) * 128)) + pages_elem_offset) + ((convert_int(get_local_id(0))) & 127))] = pages[(((((((src_pos >> 4) * 65536) + (((convert_int(get_group_id(0))) & 7) * 4096)) + (((convert_int(get_local_id(0))) >> 7) * 2048)) + ((src_pos & 15) * 128)) + pages_elem_offset) + ((convert_int(get_local_id(0))) & 127))]; + pages[((((((((dst_pos >> 4) * 65536) + (((convert_int(get_group_id(0))) & 7) * 4096)) + (((convert_int(get_local_id(0))) >> 7) * 2048)) + ((dst_pos & 15) * 128)) + pages_elem_offset) + ((convert_int(get_local_id(0))) & 127)) + 32768)] = pages[((((((((src_pos >> 4) * 65536) + (((convert_int(get_group_id(0))) & 7) * 4096)) + (((convert_int(get_local_id(0))) >> 7) * 2048)) + ((src_pos & 15) * 128)) + pages_elem_offset) + ((convert_int(get_local_id(0))) & 127)) + 32768)]; + } + } +} + +// Function: gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len) { + float NT_matmul_intermediate_pad_rf_local[32]; + float NT_matmul_intermediate_pad_rf_local_1[4]; + float NT_matmul_intermediate_pad_local[4]; + __local float red_buf0[64]; + for (int ax0_1_init = 0; ax0_1_init < 2; ++ax0_1_init) { + for (int ax1_fused_2_init = 0; ax1_fused_2_init < 2; ++ax1_fused_2_init) { + int8 v_ = (int8)((((ax0_1_init * 2) + ax1_fused_2_init))+(4*0), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*1), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*2), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*3), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*4), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*5), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*6), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*7)); + float8 v__1 = (float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f); + NT_matmul_intermediate_pad_rf_local[v_.s0] = v__1.s0; + NT_matmul_intermediate_pad_rf_local[v_.s1] = v__1.s1; + NT_matmul_intermediate_pad_rf_local[v_.s2] = v__1.s2; + NT_matmul_intermediate_pad_rf_local[v_.s3] = v__1.s3; + NT_matmul_intermediate_pad_rf_local[v_.s4] = v__1.s4; + NT_matmul_intermediate_pad_rf_local[v_.s5] = v__1.s5; + NT_matmul_intermediate_pad_rf_local[v_.s6] = v__1.s6; + NT_matmul_intermediate_pad_rf_local[v_.s7] = v__1.s7; + } + } + for (int ax2_fused_0 = 0; ax2_fused_0 < 32; ++ax2_fused_0) { + if (((convert_int(get_group_id(1))) * 2) < seq_len) { + int8 v__2 = (int8)((0)+(4*0), (0)+(4*1), (0)+(4*2), (0)+(4*3), (0)+(4*4), (0)+(4*5), (0)+(4*6), (0)+(4*7)); + float8 v__3 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__2.s0],NT_matmul_intermediate_pad_rf_local[v__2.s1],NT_matmul_intermediate_pad_rf_local[v__2.s2],NT_matmul_intermediate_pad_rf_local[v__2.s3],NT_matmul_intermediate_pad_rf_local[v__2.s4],NT_matmul_intermediate_pad_rf_local[v__2.s5],NT_matmul_intermediate_pad_rf_local[v__2.s6],NT_matmul_intermediate_pad_rf_local[v__2.s7])) + ((convert_float8(vload8(0, astype98 + ((((convert_int(get_group_id(1))) * 16384) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_weight3 + (((((convert_int(get_group_id(0))) * 32768) + ((convert_int(get_local_id(0))) * 16384)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))))); + NT_matmul_intermediate_pad_rf_local[v__2.s0] = v__3.s0; + NT_matmul_intermediate_pad_rf_local[v__2.s1] = v__3.s1; + NT_matmul_intermediate_pad_rf_local[v__2.s2] = v__3.s2; + NT_matmul_intermediate_pad_rf_local[v__2.s3] = v__3.s3; + NT_matmul_intermediate_pad_rf_local[v__2.s4] = v__3.s4; + NT_matmul_intermediate_pad_rf_local[v__2.s5] = v__3.s5; + NT_matmul_intermediate_pad_rf_local[v__2.s6] = v__3.s6; + NT_matmul_intermediate_pad_rf_local[v__2.s7] = v__3.s7; + int8 v__4 = (int8)((1)+(4*0), (1)+(4*1), (1)+(4*2), (1)+(4*3), (1)+(4*4), (1)+(4*5), (1)+(4*6), (1)+(4*7)); + float8 v__5 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__4.s0],NT_matmul_intermediate_pad_rf_local[v__4.s1],NT_matmul_intermediate_pad_rf_local[v__4.s2],NT_matmul_intermediate_pad_rf_local[v__4.s3],NT_matmul_intermediate_pad_rf_local[v__4.s4],NT_matmul_intermediate_pad_rf_local[v__4.s5],NT_matmul_intermediate_pad_rf_local[v__4.s6],NT_matmul_intermediate_pad_rf_local[v__4.s7])) + ((convert_float8(vload8(0, astype98 + ((((convert_int(get_group_id(1))) * 16384) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_weight3 + ((((((convert_int(get_group_id(0))) * 32768) + ((convert_int(get_local_id(0))) * 16384)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 8192))))); + NT_matmul_intermediate_pad_rf_local[v__4.s0] = v__5.s0; + NT_matmul_intermediate_pad_rf_local[v__4.s1] = v__5.s1; + NT_matmul_intermediate_pad_rf_local[v__4.s2] = v__5.s2; + NT_matmul_intermediate_pad_rf_local[v__4.s3] = v__5.s3; + NT_matmul_intermediate_pad_rf_local[v__4.s4] = v__5.s4; + NT_matmul_intermediate_pad_rf_local[v__4.s5] = v__5.s5; + NT_matmul_intermediate_pad_rf_local[v__4.s6] = v__5.s6; + NT_matmul_intermediate_pad_rf_local[v__4.s7] = v__5.s7; + } + if ((((convert_int(get_group_id(1))) * 2) + 1) < seq_len) { + int8 v__6 = (int8)((2)+(4*0), (2)+(4*1), (2)+(4*2), (2)+(4*3), (2)+(4*4), (2)+(4*5), (2)+(4*6), (2)+(4*7)); + float8 v__7 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__6.s0],NT_matmul_intermediate_pad_rf_local[v__6.s1],NT_matmul_intermediate_pad_rf_local[v__6.s2],NT_matmul_intermediate_pad_rf_local[v__6.s3],NT_matmul_intermediate_pad_rf_local[v__6.s4],NT_matmul_intermediate_pad_rf_local[v__6.s5],NT_matmul_intermediate_pad_rf_local[v__6.s6],NT_matmul_intermediate_pad_rf_local[v__6.s7])) + ((convert_float8(vload8(0, astype98 + (((((convert_int(get_group_id(1))) * 16384) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 8192)))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_weight3 + (((((convert_int(get_group_id(0))) * 32768) + ((convert_int(get_local_id(0))) * 16384)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))))); + NT_matmul_intermediate_pad_rf_local[v__6.s0] = v__7.s0; + NT_matmul_intermediate_pad_rf_local[v__6.s1] = v__7.s1; + NT_matmul_intermediate_pad_rf_local[v__6.s2] = v__7.s2; + NT_matmul_intermediate_pad_rf_local[v__6.s3] = v__7.s3; + NT_matmul_intermediate_pad_rf_local[v__6.s4] = v__7.s4; + NT_matmul_intermediate_pad_rf_local[v__6.s5] = v__7.s5; + NT_matmul_intermediate_pad_rf_local[v__6.s6] = v__7.s6; + NT_matmul_intermediate_pad_rf_local[v__6.s7] = v__7.s7; + int8 v__8 = (int8)((3)+(4*0), (3)+(4*1), (3)+(4*2), (3)+(4*3), (3)+(4*4), (3)+(4*5), (3)+(4*6), (3)+(4*7)); + float8 v__9 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__8.s0],NT_matmul_intermediate_pad_rf_local[v__8.s1],NT_matmul_intermediate_pad_rf_local[v__8.s2],NT_matmul_intermediate_pad_rf_local[v__8.s3],NT_matmul_intermediate_pad_rf_local[v__8.s4],NT_matmul_intermediate_pad_rf_local[v__8.s5],NT_matmul_intermediate_pad_rf_local[v__8.s6],NT_matmul_intermediate_pad_rf_local[v__8.s7])) + ((convert_float8(vload8(0, astype98 + (((((convert_int(get_group_id(1))) * 16384) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 8192)))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_weight3 + ((((((convert_int(get_group_id(0))) * 32768) + ((convert_int(get_local_id(0))) * 16384)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 8192))))); + NT_matmul_intermediate_pad_rf_local[v__8.s0] = v__9.s0; + NT_matmul_intermediate_pad_rf_local[v__8.s1] = v__9.s1; + NT_matmul_intermediate_pad_rf_local[v__8.s2] = v__9.s2; + NT_matmul_intermediate_pad_rf_local[v__8.s3] = v__9.s3; + NT_matmul_intermediate_pad_rf_local[v__8.s4] = v__9.s4; + NT_matmul_intermediate_pad_rf_local[v__8.s5] = v__9.s5; + NT_matmul_intermediate_pad_rf_local[v__8.s6] = v__9.s6; + NT_matmul_intermediate_pad_rf_local[v__8.s7] = v__9.s7; + } + } + for (int ax3_fused_2_0 = 0; ax3_fused_2_0 < 1; ++ax3_fused_2_0) { + for (int ax2 = 0; ax2 < 2; ++ax2) { + vstore2(((float2)(0.000000e+00f, 0.000000e+00f)), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + (ax2 * 2))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 4))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 8))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 12))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 16))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 20))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 24))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 28))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + } + } + for (int ax2_fused_2 = 0; ax2_fused_2 < 2; ++ax2_fused_2) { + for (int ax1 = 0; ax1 < 2; ++ax1) { + barrier(CLK_LOCAL_MEM_FENCE); + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_intermediate_pad_rf_local_1[((ax1 * 2) + ax2_fused_2)]; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 16) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 8) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 4) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 2) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + NT_matmul_intermediate_pad_local[((ax1 * 2) + ax2_fused_2)] = ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) * 32)]; + } + } + for (int ax0 = 0; ax0 < 2; ++ax0) { + for (int ax1_fused_2 = 0; ax1_fused_2 < 2; ++ax1_fused_2) { + if (((bool)(((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)(((bool)(((convert_int(get_group_id(1))) - ((seq_len + 1) >> 1)) < 0)) || ((bool)((convert_int(get_group_id(1))) == 0)))))) && ((bool)((((convert_int(get_group_id(1))) * 2) + ax0) < seq_len))) { + compute[((((((convert_int(get_group_id(1))) * 4096) + (ax0 * 2048)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] = (convert_half((NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias3[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]))); + } + } + } +} + +// Function: gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len) { + int tile_id[1]; + int batch_idx[1]; + int batch_rows[1]; + int batch_tiles[1]; + int kv_chunk_len[1]; + __local float m_smem[32]; + __local float d_smem[32]; + float O_local[32]; + __local half Q_smem[4096]; + __local half K_smem[2048]; + __local half V_smem[2048]; + float S_local[4]; + float m_prev[1]; + float m_new[1]; + float d_new[1]; + __local float m_prev_smem[32]; + tile_id[0] = (convert_int(get_group_id(0))); + batch_idx[0] = 0; + batch_rows[0] = (q_indptr[(q_indptr_elem_offset + 1)] - q_indptr[q_indptr_elem_offset]); + batch_tiles[0] = ((batch_rows[0] + 31) >> 5); + while (1) { + if (!(((batch_idx[0] < batch_size)))) { break; } + while (1) { + if (!((((bool)(batch_tiles[0] <= tile_id[0])) && ((bool)(batch_idx[0] < batch_size))))) { break; } + tile_id[0] = (tile_id[0] - batch_tiles[0]); + batch_idx[0] = (batch_idx[0] + 1); + if (batch_idx[0] < batch_size) { + int b_idx = batch_idx[0]; + batch_rows[0] = (q_indptr[((b_idx + q_indptr_elem_offset) + 1)] - q_indptr[(b_idx + q_indptr_elem_offset)]); + batch_tiles[0] = ((batch_rows[0] + 31) >> 5); + } + } + if ((batch_idx[0] < batch_size)) { + int b_idx_1 = batch_idx[0]; + int LH_start = (tile_id[0] * 32); + int q_indptr_val = q_indptr[(b_idx_1 + q_indptr_elem_offset)]; + int cur_page_indptr_begin = page_indptr[(b_idx_1 + page_indptr_elem_offset)]; + int cur_page_indptr_end = page_indptr[((b_idx_1 + page_indptr_elem_offset) + 1)]; + int condval; + if ((cur_page_indptr_begin != cur_page_indptr_end)) { + condval = ((((((cur_page_indptr_end * 16) + length_info[(b_idx_1 + length_info_elem_offset)]) + length_info[(((batch_size * 2) + b_idx_1) + length_info_elem_offset)]) - length_info[((batch_size + b_idx_1) + length_info_elem_offset)]) - (cur_page_indptr_begin * 16)) - 16); + } else { + condval = 0; + } + kv_chunk_len[0] = condval; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + m_smem[(convert_int(get_local_id(0)))] = -5.000000e+04f; + d_smem[(convert_int(get_local_id(0)))] = 1.000000e+00f; + } + for (int li_1 = 0; li_1 < 4; ++li_1) { + for (int lj_1 = 0; lj_1 < 8; ++lj_1) { + O_local[((li_1 * 8) + lj_1)] = 0.000000e+00f; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + for (int li_lj_fused_0 = 0; li_lj_fused_0 < 8; ++li_lj_fused_0) { + if (((((li_lj_fused_0 * 4) + q_indptr_val) + LH_start) + (convert_int(get_local_id(1)))) < q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)]) { + half4 condval_1; + if ((rotary_mode == 1)) { + float4 freq = (((float4)(((convert_float(q_rope_position[(((((li_lj_fused_0 * 4) + (convert_int(get_local_id(1)))) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[(((((li_lj_fused_0 * 4) + (convert_int(get_local_id(1)))) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[(((((li_lj_fused_0 * 4) + (convert_int(get_local_id(1)))) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[(((((li_lj_fused_0 * 4) + (convert_int(get_local_id(1)))) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale))) / pow(((float4)(rope_theta, rope_theta, rope_theta, rope_theta)), ((convert_float4(((int4)(((((convert_int(get_local_id(0))) & 15) * 8))+(2*0), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*1), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*2), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*3))))) * ((float4)(7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f))))); + half4 condval_2; + if (((convert_int(get_local_id(0))) < 16)) { + condval_2 = (vload4(0, q + (((((((li_lj_fused_0 * 8192) + ((convert_int(get_local_id(1))) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + 64)) * ((half4)((half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f))); + } else { + condval_2 = vload4(0, q + (((((((li_lj_fused_0 * 8192) + ((convert_int(get_local_id(1))) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) - 64)); + } + condval_1 = (convert_half4(((cos(freq) * (convert_float4(vload4(0, q + ((((((li_lj_fused_0 * 8192) + ((convert_int(get_local_id(1))) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)))))) + (sin(freq) * (convert_float4(condval_2)))))); + } else { + condval_1 = vload4(0, q + ((((((li_lj_fused_0 * 8192) + ((convert_int(get_local_id(1))) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); + } + vstore4(condval_1, 0, Q_smem + (((li_lj_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); + } else { + vstore4(((half4)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, Q_smem + (((li_lj_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); + } + } + barrier(CLK_LOCAL_MEM_FENCE); + for (int iterator = 0; iterator < ((kv_chunk_len[0] + 15) >> 4); ++iterator) { + barrier(CLK_LOCAL_MEM_FENCE); + for (int lz_ly_fused_0 = 0; lz_ly_fused_0 < 4; ++lz_ly_fused_0) { + if ((((iterator * 16) + (lz_ly_fused_0 * 4)) + (convert_int(get_local_id(1)))) < kv_chunk_len[0]) { + int condval_3; + if (((((iterator * 16) + (lz_ly_fused_0 * 4)) + (convert_int(get_local_id(1)))) < length_info[(((batch_size * 2) + b_idx_1) + length_info_elem_offset)])) { + condval_3 = (((iterator * 16) + (lz_ly_fused_0 * 4)) + (convert_int(get_local_id(1)))); + } else { + condval_3 = (((((iterator * 16) + (lz_ly_fused_0 * 4)) + (convert_int(get_local_id(1)))) + length_info[((batch_size + b_idx_1) + length_info_elem_offset)]) - length_info[(((batch_size * 2) + b_idx_1) + length_info_elem_offset)]); + } + int seq_offset = condval_3; + int page_no = page_values[(((seq_offset >> 4) + cur_page_indptr_begin) + page_values_elem_offset)]; + half4 condval_4; + if ((rotary_mode == 1)) { + float4 freq_1 = (((float4)(((convert_float(((((iterator * 16) + (lz_ly_fused_0 * 4)) + k_rope_pos_offset[(b_idx_1 + k_rope_pos_offset_elem_offset)]) + (convert_int(get_local_id(1)))))) * rope_scale), ((convert_float(((((iterator * 16) + (lz_ly_fused_0 * 4)) + k_rope_pos_offset[(b_idx_1 + k_rope_pos_offset_elem_offset)]) + (convert_int(get_local_id(1)))))) * rope_scale), ((convert_float(((((iterator * 16) + (lz_ly_fused_0 * 4)) + k_rope_pos_offset[(b_idx_1 + k_rope_pos_offset_elem_offset)]) + (convert_int(get_local_id(1)))))) * rope_scale), ((convert_float(((((iterator * 16) + (lz_ly_fused_0 * 4)) + k_rope_pos_offset[(b_idx_1 + k_rope_pos_offset_elem_offset)]) + (convert_int(get_local_id(1)))))) * rope_scale))) / pow(((float4)(rope_theta, rope_theta, rope_theta, rope_theta)), ((convert_float4(((int4)(((((convert_int(get_local_id(0))) & 15) * 8))+(2*0), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*1), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*2), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*3))))) * ((float4)(7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f))))); + int4 v_ = (int4)(((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*0), ((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*1), ((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*2), ((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*3)); + half4 condval_5; + if (((convert_int(get_local_id(0))) < 16)) { + int4 v__1 = (int4)((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 64))+(1*0), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 64))+(1*1), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 64))+(1*2), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 64))+(1*3)); + condval_5 = (((half4)(pages[v__1.s0],pages[v__1.s1],pages[v__1.s2],pages[v__1.s3])) * ((half4)((half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f))); + } else { + int4 v__2 = (int4)((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) - 64))+(1*0), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) - 64))+(1*1), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) - 64))+(1*2), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) - 64))+(1*3)); + condval_5 = ((half4)(pages[v__2.s0],pages[v__2.s1],pages[v__2.s2],pages[v__2.s3])); + } + condval_4 = (convert_half4(((cos(freq_1) * (convert_float4(((half4)(pages[v_.s0],pages[v_.s1],pages[v_.s2],pages[v_.s3]))))) + (sin(freq_1) * (convert_float4(condval_5)))))); + } else { + int4 v__3 = (int4)(((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*0), ((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*1), ((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*2), ((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*3)); + condval_4 = ((half4)(pages[v__3.s0],pages[v__3.s1],pages[v__3.s2],pages[v__3.s3])); + } + vstore4(condval_4, 0, K_smem + (((lz_ly_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); + } else { + vstore4(((half4)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, K_smem + (((lz_ly_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); + } + } + barrier(CLK_LOCAL_MEM_FENCE); + for (int lz_ly_fused_0_1 = 0; lz_ly_fused_0_1 < 4; ++lz_ly_fused_0_1) { + if ((((iterator * 16) + (lz_ly_fused_0_1 * 4)) + (convert_int(get_local_id(1)))) < kv_chunk_len[0]) { + int condval_6; + if (((((iterator * 16) + (lz_ly_fused_0_1 * 4)) + (convert_int(get_local_id(1)))) < length_info[(((batch_size * 2) + b_idx_1) + length_info_elem_offset)])) { + condval_6 = (((iterator * 16) + (lz_ly_fused_0_1 * 4)) + (convert_int(get_local_id(1)))); + } else { + condval_6 = (((((iterator * 16) + (lz_ly_fused_0_1 * 4)) + (convert_int(get_local_id(1)))) + length_info[((batch_size + b_idx_1) + length_info_elem_offset)]) - length_info[(((batch_size * 2) + b_idx_1) + length_info_elem_offset)]); + } + int seq_offset_1 = condval_6; + int page_no_1 = page_values[(((seq_offset_1 >> 4) + cur_page_indptr_begin) + page_values_elem_offset)]; + int4 v__4 = (int4)((((((((page_no_1 * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset_1 & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 32768))+(1*0), (((((((page_no_1 * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset_1 & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 32768))+(1*1), (((((((page_no_1 * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset_1 & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 32768))+(1*2), (((((((page_no_1 * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset_1 & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 32768))+(1*3)); + vstore4(((half4)(pages[v__4.s0],pages[v__4.s1],pages[v__4.s2],pages[v__4.s3])), 0, V_smem + (((lz_ly_fused_0_1 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); + } else { + vstore4(((half4)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, V_smem + (((lz_ly_fused_0_1 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); + } + } + barrier(CLK_LOCAL_MEM_FENCE); + for (int li_1_init = 0; li_1_init < 2; ++li_1_init) { + for (int lj_1_init = 0; lj_1_init < 2; ++lj_1_init) { + S_local[((li_1_init * 2) + lj_1_init)] = 0.000000e+00f; + } + } + for (int lk_0 = 0; lk_0 < 16; ++lk_0) { + for (int li_1_1 = 0; li_1_1 < 2; ++li_1_1) { + for (int lj_1_1 = 0; lj_1_1 < 2; ++lj_1_1) { + for (int lk_1 = 0; lk_1 < 8; ++lk_1) { + S_local[((li_1_1 * 2) + lj_1_1)] = (S_local[((li_1_1 * 2) + lj_1_1)] + ((((convert_float(Q_smem[((((((convert_int(get_local_id(1))) * 1024) + (((convert_int(get_local_id(0))) >> 3) * 256)) + (li_1_1 * 128)) + (lk_0 * 8)) + lk_1)])) * (convert_float(K_smem[((((((convert_int(get_local_id(0))) & 7) * 256) + (lj_1_1 * 128)) + (lk_0 * 8)) + lk_1)]))) * attn_score_scaling_factor) * 1.275174e-01f)); + } + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + for (int li_1_2 = 0; li_1_2 < 2; ++li_1_2) { + for (int lj_1_2 = 0; lj_1_2 < 2; ++lj_1_2) { + ((__local float*)K_smem)[((((((convert_int(get_local_id(1))) * 128) + (((convert_int(get_local_id(0))) >> 3) * 32)) + (li_1_2 * 16)) + (((convert_int(get_local_id(0))) & 7) * 2)) + lj_1_2)] = S_local[((li_1_2 * 2) + lj_1_2)]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + m_prev[0] = m_smem[(convert_int(get_local_id(0)))]; + m_new[0] = m_smem[(convert_int(get_local_id(0)))]; + for (int j = 0; j < 16; ++j) { + int cse_var_4 = ((iterator * 16) + j); + bool condval_7; + if ((0 < causal)) { + condval_7 = (cse_var_4 <= (((((((convert_int(get_local_id(1))) * 32) + kv_chunk_len[0]) + q_indptr[(b_idx_1 + q_indptr_elem_offset)]) + LH_start) + (convert_int(get_local_id(0)))) - q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)])); + } else { + condval_7 = (cse_var_4 < kv_chunk_len[0]); + } + if (condval_7) { + m_new[0] = max((float)m_new[0], (float)((__local float*)K_smem)[((((convert_int(get_local_id(1))) * 512) + ((convert_int(get_local_id(0))) * 16)) + j)]); + } + } + d_new[0] = (d_smem[(convert_int(get_local_id(0)))] * exp2((m_prev[0] - m_new[0]))); + } + for (int j_1 = 0; j_1 < 16; ++j_1) { + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + int cse_var_5 = ((iterator * 16) + j_1); + bool condval_8; + if ((0 < causal)) { + condval_8 = (cse_var_5 <= (((((((convert_int(get_local_id(1))) * 32) + kv_chunk_len[0]) + q_indptr[(b_idx_1 + q_indptr_elem_offset)]) + LH_start) + (convert_int(get_local_id(0)))) - q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)])); + } else { + condval_8 = (cse_var_5 < kv_chunk_len[0]); + } + if (condval_8) { + ((__local float*)K_smem)[((((convert_int(get_local_id(1))) * 512) + ((convert_int(get_local_id(0))) * 16)) + j_1)] = exp2((((__local float*)K_smem)[((((convert_int(get_local_id(1))) * 512) + ((convert_int(get_local_id(0))) * 16)) + j_1)] - m_new[0])); + } else { + ((__local float*)K_smem)[(((convert_int(get_local_id(0))) * 16) + j_1)] = exp2((-5.000000e+04f - m_new[0])); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + for (int j_2 = 0; j_2 < 16; ++j_2) { + d_new[0] = (d_new[0] + ((__local float*)K_smem)[(((convert_int(get_local_id(0))) * 16) + j_2)]); + } + m_smem[(convert_int(get_local_id(0)))] = m_new[0]; + d_smem[(convert_int(get_local_id(0)))] = d_new[0]; + m_prev_smem[(convert_int(get_local_id(0)))] = m_prev[0]; + } + barrier(CLK_LOCAL_MEM_FENCE); + for (int li_1_init_1 = 0; li_1_init_1 < 4; ++li_1_init_1) { + for (int lj_1_init_1 = 0; lj_1_init_1 < 8; ++lj_1_init_1) { + O_local[((li_1_init_1 * 8) + lj_1_init_1)] = (O_local[((li_1_init_1 * 8) + lj_1_init_1)] * exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_init_1)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_init_1)]))); + } + } + for (int lk_0_1 = 0; lk_0_1 < 2; ++lk_0_1) { + for (int lk_1_1 = 0; lk_1_1 < 8; ++lk_1_1) { + for (int li_1_3 = 0; li_1_3 < 4; ++li_1_3) { + for (int lj_1_3 = 0; lj_1_3 < 8; ++lj_1_3) { + O_local[((li_1_3 * 8) + lj_1_3)] = (O_local[((li_1_3 * 8) + lj_1_3)] + (((__local float*)K_smem)[((((((convert_int(get_local_id(1))) * 128) + (((convert_int(get_local_id(0))) >> 4) * 64)) + (li_1_3 * 16)) + (lk_0_1 * 8)) + lk_1_1)] * (convert_float(V_smem[((((lk_0_1 * 1024) + (lk_1_1 * 128)) + (((convert_int(get_local_id(0))) & 15) * 8)) + lj_1_3)])))); + } + } + } + } + } + for (int li_1_4 = 0; li_1_4 < 4; ++li_1_4) { + for (int lj_1_4 = 0; lj_1_4 < 8; ++lj_1_4) { + int cur_L = ((((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + q_indptr[(b_idx_1 + q_indptr_elem_offset)]) + LH_start) + li_1_4); + if (cur_L < q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)]) { + output[((((cur_L * 2048) + ((convert_int(get_group_id(1))) * 128)) + (((convert_int(get_local_id(0))) & 15) * 8)) + lj_1_4)] = (convert_half((O_local[((li_1_4 * 8) + lj_1_4)] / d_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_4)]))); + } + } + } + if ((convert_int(get_local_id(1))) < 1) { + int cur_L_1 = (((((convert_int(get_local_id(1))) * 32) + q_indptr[(b_idx_1 + q_indptr_elem_offset)]) + LH_start) + (convert_int(get_local_id(0)))); + if (cur_L_1 < q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)]) { + lse[((cur_L_1 * 16) + (convert_int(get_group_id(1))))] = (m_smem[(convert_int(get_local_id(0)))] + log2(d_smem[(convert_int(get_local_id(0)))])); + } + } + tile_id[0] = (tile_id[0] + 16); + } + } +} + +// Function: gpt_neox_q0f16_batch_prefill_ragged_kv_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode) { + int tile_id[1]; + int batch_idx[1]; + int batch_rows[1]; + int batch_tiles[1]; + int kv_chunk_len[1]; + __local float m_smem[32]; + __local float d_smem[32]; + float8 O_local[4]; + __local half Q_smem[4096]; + __local half K_smem[4096]; + __local half8 V_smem[512]; + float4 S_local[2]; + __local float S_smem[1024]; + float m_prev[1]; + float m_new[1]; + float d_new[1]; + __local float m_prev_smem[32]; + tile_id[0] = (convert_int(get_group_id(0))); + batch_idx[0] = 0; + batch_rows[0] = (q_indptr[(q_indptr_elem_offset + 1)] - q_indptr[q_indptr_elem_offset]); + batch_tiles[0] = ((batch_rows[0] + 31) >> 5); + while (1) { + if (!(((batch_idx[0] < batch_size)))) { break; } + while (1) { + if (!((((bool)(batch_tiles[0] <= tile_id[0])) && ((bool)(batch_idx[0] < batch_size))))) { break; } + tile_id[0] = (tile_id[0] - batch_tiles[0]); + batch_idx[0] = (batch_idx[0] + 1); + if (batch_idx[0] < batch_size) { + int b_idx = batch_idx[0]; + batch_rows[0] = (q_indptr[((b_idx + q_indptr_elem_offset) + 1)] - q_indptr[(b_idx + q_indptr_elem_offset)]); + batch_tiles[0] = ((batch_rows[0] + 31) >> 5); + } + } + if ((batch_idx[0] < batch_size)) { + int b_idx_1 = batch_idx[0]; + int q_indptr_val = q_indptr[(b_idx_1 + q_indptr_elem_offset)]; + int LH_start = (tile_id[0] * 32); + kv_chunk_len[0] = (kv_indptr[((b_idx_1 + kv_indptr_elem_offset) + 1)] - kv_indptr[(b_idx_1 + kv_indptr_elem_offset)]); + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + m_smem[(convert_int(get_local_id(0)))] = -5.000000e+04f; + d_smem[(convert_int(get_local_id(0)))] = 1.000000e+00f; + } + for (int li_1 = 0; li_1 < 4; ++li_1) { + O_local[li_1] = ((float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)); + } + barrier(CLK_LOCAL_MEM_FENCE); + for (int li_0 = 0; li_0 < 2; ++li_0) { + for (int lj_0_0 = 0; lj_0_0 < 2; ++lj_0_0) { + if ((((((li_0 * 16) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + q_indptr_val) + LH_start) < q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)]) { + half8 condval; + if ((rotary_mode == 1)) { + float8 freq = (((float8)(((convert_float(q_rope_position[((((((li_0 * 16) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[((((((li_0 * 16) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[((((((li_0 * 16) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[((((((li_0 * 16) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[((((((li_0 * 16) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[((((((li_0 * 16) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[((((((li_0 * 16) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[((((((li_0 * 16) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale))) / pow(((float8)(rope_theta, rope_theta, rope_theta, rope_theta, rope_theta, rope_theta, rope_theta, rope_theta)), ((convert_float8(((int8)(((((convert_int(get_local_id(0))) & 7) * 16))+(2*0), ((((convert_int(get_local_id(0))) & 7) * 16))+(2*1), ((((convert_int(get_local_id(0))) & 7) * 16))+(2*2), ((((convert_int(get_local_id(0))) & 7) * 16))+(2*3), ((((convert_int(get_local_id(0))) & 7) * 16))+(2*4), ((((convert_int(get_local_id(0))) & 7) * 16))+(2*5), ((((convert_int(get_local_id(0))) & 7) * 16))+(2*6), ((((convert_int(get_local_id(0))) & 7) * 16))+(2*7))))) * ((float8)(7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f))))); + half8 condval_1; + if ((lj_0_0 < 1)) { + condval_1 = (vload8(0, q + (((((((((li_0 * 32768) + ((convert_int(get_local_id(1))) * 8192)) + (((convert_int(get_local_id(0))) >> 3) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + (lj_0_0 * 64)) + (((convert_int(get_local_id(0))) & 7) * 8)) + 64)) * ((half8)((half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f))); + } else { + condval_1 = vload8(0, q + (((((((((li_0 * 32768) + ((convert_int(get_local_id(1))) * 8192)) + (((convert_int(get_local_id(0))) >> 3) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + (lj_0_0 * 64)) + (((convert_int(get_local_id(0))) & 7) * 8)) - 64)); + } + condval = (convert_half8(((cos(freq) * (convert_float8(vload8(0, q + ((((((((li_0 * 32768) + ((convert_int(get_local_id(1))) * 8192)) + (((convert_int(get_local_id(0))) >> 3) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + (lj_0_0 * 64)) + (((convert_int(get_local_id(0))) & 7) * 8)))))) + (sin(freq) * (convert_float8(condval_1)))))); + } else { + condval = vload8(0, q + ((((((((li_0 * 32768) + ((convert_int(get_local_id(1))) * 8192)) + (((convert_int(get_local_id(0))) >> 3) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + (lj_0_0 * 64)) + (((convert_int(get_local_id(0))) & 7) * 8))); + } + vstore8(condval, 0, Q_smem + (((((li_0 * 2048) + ((convert_int(get_local_id(1))) * 512)) + (((convert_int(get_local_id(0))) >> 3) * 128)) + (lj_0_0 * 64)) + (((convert_int(get_local_id(0))) & 7) * 8))); + } else { + vstore8(((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, Q_smem + (((((li_0 * 2048) + ((convert_int(get_local_id(1))) * 512)) + (((convert_int(get_local_id(0))) >> 3) * 128)) + (lj_0_0 * 64)) + (((convert_int(get_local_id(0))) & 7) * 8))); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + for (int iterator = 0; iterator < ((kv_chunk_len[0] + 31) >> 5); ++iterator) { + int L_kv_base = kv_indptr[(b_idx_1 + kv_indptr_elem_offset)]; + for (int lz_0 = 0; lz_0 < 2; ++lz_0) { + for (int ly_0_0 = 0; ly_0_0 < 2; ++ly_0_0) { + if (((((iterator * 32) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) < kv_chunk_len[0]) { + int8 v_ = (int8)(((((((ly_0_0 * 2048) + (((convert_int(get_local_id(0))) & 7) * 256)) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)))+(32*0), ((((((ly_0_0 * 2048) + (((convert_int(get_local_id(0))) & 7) * 256)) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)))+(32*1), ((((((ly_0_0 * 2048) + (((convert_int(get_local_id(0))) & 7) * 256)) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)))+(32*2), ((((((ly_0_0 * 2048) + (((convert_int(get_local_id(0))) & 7) * 256)) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)))+(32*3), ((((((ly_0_0 * 2048) + (((convert_int(get_local_id(0))) & 7) * 256)) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)))+(32*4), ((((((ly_0_0 * 2048) + (((convert_int(get_local_id(0))) & 7) * 256)) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)))+(32*5), ((((((ly_0_0 * 2048) + (((convert_int(get_local_id(0))) & 7) * 256)) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)))+(32*6), ((((((ly_0_0 * 2048) + (((convert_int(get_local_id(0))) & 7) * 256)) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)))+(32*7)); + half8 condval_2; + if ((rotary_mode == 1)) { + float8 freq_1 = (((float8)(((convert_float((((((iterator * 32) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + k_rope_pos_offset[(b_idx_1 + k_rope_pos_offset_elem_offset)]))) * rope_scale), ((convert_float((((((iterator * 32) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + k_rope_pos_offset[(b_idx_1 + k_rope_pos_offset_elem_offset)]))) * rope_scale), ((convert_float((((((iterator * 32) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + k_rope_pos_offset[(b_idx_1 + k_rope_pos_offset_elem_offset)]))) * rope_scale), ((convert_float((((((iterator * 32) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + k_rope_pos_offset[(b_idx_1 + k_rope_pos_offset_elem_offset)]))) * rope_scale), ((convert_float((((((iterator * 32) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + k_rope_pos_offset[(b_idx_1 + k_rope_pos_offset_elem_offset)]))) * rope_scale), ((convert_float((((((iterator * 32) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + k_rope_pos_offset[(b_idx_1 + k_rope_pos_offset_elem_offset)]))) * rope_scale), ((convert_float((((((iterator * 32) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + k_rope_pos_offset[(b_idx_1 + k_rope_pos_offset_elem_offset)]))) * rope_scale), ((convert_float((((((iterator * 32) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + k_rope_pos_offset[(b_idx_1 + k_rope_pos_offset_elem_offset)]))) * rope_scale))) / pow(((float8)(rope_theta, rope_theta, rope_theta, rope_theta, rope_theta, rope_theta, rope_theta, rope_theta)), ((convert_float8(((int8)(((((convert_int(get_local_id(0))) & 7) * 16))+(2*0), ((((convert_int(get_local_id(0))) & 7) * 16))+(2*1), ((((convert_int(get_local_id(0))) & 7) * 16))+(2*2), ((((convert_int(get_local_id(0))) & 7) * 16))+(2*3), ((((convert_int(get_local_id(0))) & 7) * 16))+(2*4), ((((convert_int(get_local_id(0))) & 7) * 16))+(2*5), ((((convert_int(get_local_id(0))) & 7) * 16))+(2*6), ((((convert_int(get_local_id(0))) & 7) * 16))+(2*7))))) * ((float8)(7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f))))); + half8 condval_3; + if ((ly_0_0 < 1)) { + condval_3 = (vload8(0, k + (((((((((iterator * 65536) + (lz_0 * 32768)) + ((convert_int(get_local_id(1))) * 8192)) + (((convert_int(get_local_id(0))) >> 3) * 2048)) + (L_kv_base * 2048)) + ((convert_int(get_group_id(1))) * 128)) + (ly_0_0 * 64)) + (((convert_int(get_local_id(0))) & 7) * 8)) + 64)) * ((half8)((half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f))); + } else { + condval_3 = vload8(0, k + (((((((((iterator * 65536) + (lz_0 * 32768)) + ((convert_int(get_local_id(1))) * 8192)) + (((convert_int(get_local_id(0))) >> 3) * 2048)) + (L_kv_base * 2048)) + ((convert_int(get_group_id(1))) * 128)) + (ly_0_0 * 64)) + (((convert_int(get_local_id(0))) & 7) * 8)) - 64)); + } + condval_2 = (convert_half8(((cos(freq_1) * (convert_float8(vload8(0, k + ((((((((iterator * 65536) + (lz_0 * 32768)) + ((convert_int(get_local_id(1))) * 8192)) + (((convert_int(get_local_id(0))) >> 3) * 2048)) + (L_kv_base * 2048)) + ((convert_int(get_group_id(1))) * 128)) + (ly_0_0 * 64)) + (((convert_int(get_local_id(0))) & 7) * 8)))))) + (sin(freq_1) * (convert_float8(condval_3)))))); + } else { + condval_2 = vload8(0, k + ((((((((iterator * 65536) + (lz_0 * 32768)) + ((convert_int(get_local_id(1))) * 8192)) + (((convert_int(get_local_id(0))) >> 3) * 2048)) + (L_kv_base * 2048)) + ((convert_int(get_group_id(1))) * 128)) + (ly_0_0 * 64)) + (((convert_int(get_local_id(0))) & 7) * 8))); + } + K_smem[v_.s0] = condval_2.s0; + K_smem[v_.s1] = condval_2.s1; + K_smem[v_.s2] = condval_2.s2; + K_smem[v_.s3] = condval_2.s3; + K_smem[v_.s4] = condval_2.s4; + K_smem[v_.s5] = condval_2.s5; + K_smem[v_.s6] = condval_2.s6; + K_smem[v_.s7] = condval_2.s7; + } else { + int8 v__1 = (int8)(((((((ly_0_0 * 2048) + (((convert_int(get_local_id(0))) & 7) * 256)) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)))+(32*0), ((((((ly_0_0 * 2048) + (((convert_int(get_local_id(0))) & 7) * 256)) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)))+(32*1), ((((((ly_0_0 * 2048) + (((convert_int(get_local_id(0))) & 7) * 256)) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)))+(32*2), ((((((ly_0_0 * 2048) + (((convert_int(get_local_id(0))) & 7) * 256)) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)))+(32*3), ((((((ly_0_0 * 2048) + (((convert_int(get_local_id(0))) & 7) * 256)) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)))+(32*4), ((((((ly_0_0 * 2048) + (((convert_int(get_local_id(0))) & 7) * 256)) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)))+(32*5), ((((((ly_0_0 * 2048) + (((convert_int(get_local_id(0))) & 7) * 256)) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)))+(32*6), ((((((ly_0_0 * 2048) + (((convert_int(get_local_id(0))) & 7) * 256)) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)))+(32*7)); + half8 v__2 = (half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f); + K_smem[v__1.s0] = v__2.s0; + K_smem[v__1.s1] = v__2.s1; + K_smem[v__1.s2] = v__2.s2; + K_smem[v__1.s3] = v__2.s3; + K_smem[v__1.s4] = v__2.s4; + K_smem[v__1.s5] = v__2.s5; + K_smem[v__1.s6] = v__2.s6; + K_smem[v__1.s7] = v__2.s7; + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + for (int lz_0_1 = 0; lz_0_1 < 2; ++lz_0_1) { + for (int ly_0_0_1 = 0; ly_0_0_1 < 2; ++ly_0_0_1) { + if (((((iterator * 32) + (lz_0_1 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) < kv_chunk_len[0]) { + V_smem[(((((lz_0_1 * 256) + ((convert_int(get_local_id(1))) * 64)) + (((convert_int(get_local_id(0))) >> 3) * 16)) + (ly_0_0_1 * 8)) + ((convert_int(get_local_id(0))) & 7))] = vload8(0, v + ((((((((iterator * 65536) + (lz_0_1 * 32768)) + ((convert_int(get_local_id(1))) * 8192)) + (((convert_int(get_local_id(0))) >> 3) * 2048)) + (L_kv_base * 2048)) + ((convert_int(get_group_id(1))) * 128)) + (ly_0_0_1 * 64)) + (((convert_int(get_local_id(0))) & 7) * 8))); + } else { + V_smem[(((((lz_0_1 * 256) + ((convert_int(get_local_id(1))) * 64)) + (((convert_int(get_local_id(0))) >> 3) * 16)) + (ly_0_0_1 * 8)) + ((convert_int(get_local_id(0))) & 7))] = ((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + S_local[0] = ((float4)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)); + S_local[1] = ((float4)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)); + for (int lk_0 = 0; lk_0 < 8; ++lk_0) { + for (int lk_1 = 0; lk_1 < 16; ++lk_1) { + S_local[0] = (S_local[0] + (((((float4)((convert_float(Q_smem[(((((convert_int(get_local_id(1))) * 1024) + (((convert_int(get_local_id(0))) >> 3) * 256)) + (lk_0 * 16)) + lk_1)])), (convert_float(Q_smem[(((((convert_int(get_local_id(1))) * 1024) + (((convert_int(get_local_id(0))) >> 3) * 256)) + (lk_0 * 16)) + lk_1)])), (convert_float(Q_smem[(((((convert_int(get_local_id(1))) * 1024) + (((convert_int(get_local_id(0))) >> 3) * 256)) + (lk_0 * 16)) + lk_1)])), (convert_float(Q_smem[(((((convert_int(get_local_id(1))) * 1024) + (((convert_int(get_local_id(0))) >> 3) * 256)) + (lk_0 * 16)) + lk_1)])))) * (convert_float4(vload4(0, K_smem + (((lk_0 * 512) + (lk_1 * 32)) + (((convert_int(get_local_id(0))) & 7) * 4)))))) * ((float4)(attn_score_scaling_factor, attn_score_scaling_factor, attn_score_scaling_factor, attn_score_scaling_factor))) * ((float4)(1.275174e-01f, 1.275174e-01f, 1.275174e-01f, 1.275174e-01f)))); + } + for (int lk_1_1 = 0; lk_1_1 < 16; ++lk_1_1) { + S_local[1] = (S_local[1] + (((((float4)((convert_float(Q_smem[((((((convert_int(get_local_id(1))) * 1024) + (((convert_int(get_local_id(0))) >> 3) * 256)) + (lk_0 * 16)) + lk_1_1) + 128)])), (convert_float(Q_smem[((((((convert_int(get_local_id(1))) * 1024) + (((convert_int(get_local_id(0))) >> 3) * 256)) + (lk_0 * 16)) + lk_1_1) + 128)])), (convert_float(Q_smem[((((((convert_int(get_local_id(1))) * 1024) + (((convert_int(get_local_id(0))) >> 3) * 256)) + (lk_0 * 16)) + lk_1_1) + 128)])), (convert_float(Q_smem[((((((convert_int(get_local_id(1))) * 1024) + (((convert_int(get_local_id(0))) >> 3) * 256)) + (lk_0 * 16)) + lk_1_1) + 128)])))) * (convert_float4(vload4(0, K_smem + (((lk_0 * 512) + (lk_1_1 * 32)) + (((convert_int(get_local_id(0))) & 7) * 4)))))) * ((float4)(attn_score_scaling_factor, attn_score_scaling_factor, attn_score_scaling_factor, attn_score_scaling_factor))) * ((float4)(1.275174e-01f, 1.275174e-01f, 1.275174e-01f, 1.275174e-01f)))); + } + } + barrier(CLK_LOCAL_MEM_FENCE); + for (int li_1_1 = 0; li_1_1 < 2; ++li_1_1) { + vstore4(S_local[li_1_1], 0, S_smem + (((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 3) * 64)) + (li_1_1 * 32)) + (((convert_int(get_local_id(0))) & 7) * 4))); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + m_prev[0] = m_smem[(convert_int(get_local_id(0)))]; + m_new[0] = m_smem[(convert_int(get_local_id(0)))]; + for (int j = 0; j < 32; ++j) { + int cse_var_3 = ((iterator * 32) + j); + bool condval_4; + if ((0 < causal)) { + condval_4 = (cse_var_3 <= (((((((convert_int(get_local_id(1))) * 32) + kv_chunk_len[0]) + q_indptr[(b_idx_1 + q_indptr_elem_offset)]) + LH_start) + (convert_int(get_local_id(0)))) - q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)])); + } else { + condval_4 = (cse_var_3 < kv_chunk_len[0]); + } + if (condval_4) { + m_new[0] = max((float)m_new[0], (float)S_smem[((((convert_int(get_local_id(1))) * 1024) + ((convert_int(get_local_id(0))) * 32)) + j)]); + } + } + d_new[0] = (d_smem[(convert_int(get_local_id(0)))] * exp2((m_prev[0] - m_new[0]))); + } + for (int j_1 = 0; j_1 < 32; ++j_1) { + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + int cse_var_4 = ((iterator * 32) + j_1); + bool condval_5; + if ((0 < causal)) { + condval_5 = (cse_var_4 <= (((((((convert_int(get_local_id(1))) * 32) + kv_chunk_len[0]) + q_indptr[(b_idx_1 + q_indptr_elem_offset)]) + LH_start) + (convert_int(get_local_id(0)))) - q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)])); + } else { + condval_5 = (cse_var_4 < kv_chunk_len[0]); + } + if (condval_5) { + S_smem[((((convert_int(get_local_id(1))) * 1024) + ((convert_int(get_local_id(0))) * 32)) + j_1)] = exp2((S_smem[((((convert_int(get_local_id(1))) * 1024) + ((convert_int(get_local_id(0))) * 32)) + j_1)] - m_new[0])); + } else { + S_smem[(((convert_int(get_local_id(0))) * 32) + j_1)] = exp2((-5.000000e+04f - m_new[0])); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + for (int j_2 = 0; j_2 < 32; ++j_2) { + d_new[0] = (d_new[0] + S_smem[(((convert_int(get_local_id(0))) * 32) + j_2)]); + } + m_smem[(convert_int(get_local_id(0)))] = m_new[0]; + d_smem[(convert_int(get_local_id(0)))] = d_new[0]; + m_prev_smem[(convert_int(get_local_id(0)))] = m_prev[0]; + } + barrier(CLK_LOCAL_MEM_FENCE); + O_local[0] = (O_local[0] * ((float8)(exp2((m_prev_smem[(((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4))] - m_smem[(((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4))])), exp2((m_prev_smem[(((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4))] - m_smem[(((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4))])), exp2((m_prev_smem[(((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4))] - m_smem[(((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4))])), exp2((m_prev_smem[(((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4))] - m_smem[(((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4))])), exp2((m_prev_smem[(((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4))] - m_smem[(((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4))])), exp2((m_prev_smem[(((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4))] - m_smem[(((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4))])), exp2((m_prev_smem[(((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4))] - m_smem[(((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4))])), exp2((m_prev_smem[(((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4))] - m_smem[(((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4))]))))); + O_local[1] = (O_local[1] * ((float8)(exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 1)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 1)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 1)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 1)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 1)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 1)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 1)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 1)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 1)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 1)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 1)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 1)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 1)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 1)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 1)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 1)]))))); + O_local[2] = (O_local[2] * ((float8)(exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 2)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 2)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 2)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 2)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 2)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 2)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 2)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 2)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 2)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 2)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 2)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 2)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 2)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 2)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 2)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 2)]))))); + O_local[3] = (O_local[3] * ((float8)(exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 3)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 3)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 3)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 3)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 3)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 3)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 3)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 3)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 3)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 3)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 3)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 3)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 3)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 3)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 3)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 3)]))))); + for (int lk_0_1 = 0; lk_0_1 < 2; ++lk_0_1) { + for (int lk_1_2 = 0; lk_1_2 < 16; ++lk_1_2) { + O_local[0] = (O_local[0] + (((float8)(S_smem[(((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2)], S_smem[(((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2)], S_smem[(((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2)], S_smem[(((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2)], S_smem[(((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2)], S_smem[(((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2)], S_smem[(((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2)], S_smem[(((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2)])) * (convert_float8(V_smem[(((lk_0_1 * 256) + (lk_1_2 * 16)) + ((convert_int(get_local_id(0))) & 15))])))); + O_local[1] = (O_local[1] + (((float8)(S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 32)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 32)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 32)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 32)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 32)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 32)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 32)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 32)])) * (convert_float8(V_smem[(((lk_0_1 * 256) + (lk_1_2 * 16)) + ((convert_int(get_local_id(0))) & 15))])))); + O_local[2] = (O_local[2] + (((float8)(S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 64)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 64)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 64)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 64)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 64)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 64)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 64)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 64)])) * (convert_float8(V_smem[(((lk_0_1 * 256) + (lk_1_2 * 16)) + ((convert_int(get_local_id(0))) & 15))])))); + O_local[3] = (O_local[3] + (((float8)(S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 96)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 96)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 96)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 96)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 96)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 96)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 96)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 96)])) * (convert_float8(V_smem[(((lk_0_1 * 256) + (lk_1_2 * 16)) + ((convert_int(get_local_id(0))) & 15))])))); + } + } + } + for (int li_1_2 = 0; li_1_2 < 4; ++li_1_2) { + int cur_L = ((((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + q_indptr[(b_idx_1 + q_indptr_elem_offset)]) + LH_start) + li_1_2); + if (cur_L < q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)]) { + vstore8((convert_half8((O_local[li_1_2] / ((float8)(d_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_2)], d_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_2)], d_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_2)], d_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_2)], d_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_2)], d_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_2)], d_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_2)], d_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_2)]))))), 0, output + (((cur_L * 2048) + ((convert_int(get_group_id(1))) * 128)) + (((convert_int(get_local_id(0))) & 15) * 8))); + } + } + if ((convert_int(get_local_id(1))) < 1) { + int cur_L_1 = (((((convert_int(get_local_id(1))) * 32) + q_indptr[(b_idx_1 + q_indptr_elem_offset)]) + LH_start) + (convert_int(get_local_id(0)))); + if (cur_L_1 < q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)]) { + lse[((cur_L_1 * 16) + (convert_int(get_group_id(1))))] = (m_smem[(convert_int(get_local_id(0)))] + log2(d_smem[(convert_int(get_local_id(0)))])); + } + } + tile_id[0] = (tile_id[0] + 8); + } + } +} + +// Function: gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50) { + float NT_matmul_rf_local[8]; + half gpt_neox_layers_0_mlp_dense_h_to_4h_weight2_local[8]; + float NT_matmul_rf_local_1[1]; + __local float red_buf0[64]; + vstore8(((float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)), 0, NT_matmul_rf_local + 0); + for (int ax1_fused_u_fused_0 = 0; ax1_fused_u_fused_0 < 8; ++ax1_fused_u_fused_0) { + gpt_neox_layers_0_mlp_dense_h_to_4h_weight2_local[0] = gpt_neox_layers_0_mlp_dense_h_to_4h_weight2[(((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))]; + gpt_neox_layers_0_mlp_dense_h_to_4h_weight2_local[1] = gpt_neox_layers_0_mlp_dense_h_to_4h_weight2[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 1)]; + gpt_neox_layers_0_mlp_dense_h_to_4h_weight2_local[2] = gpt_neox_layers_0_mlp_dense_h_to_4h_weight2[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2)]; + gpt_neox_layers_0_mlp_dense_h_to_4h_weight2_local[3] = gpt_neox_layers_0_mlp_dense_h_to_4h_weight2[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 3)]; + gpt_neox_layers_0_mlp_dense_h_to_4h_weight2_local[4] = gpt_neox_layers_0_mlp_dense_h_to_4h_weight2[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 4)]; + gpt_neox_layers_0_mlp_dense_h_to_4h_weight2_local[5] = gpt_neox_layers_0_mlp_dense_h_to_4h_weight2[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 5)]; + gpt_neox_layers_0_mlp_dense_h_to_4h_weight2_local[6] = gpt_neox_layers_0_mlp_dense_h_to_4h_weight2[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 6)]; + gpt_neox_layers_0_mlp_dense_h_to_4h_weight2_local[7] = gpt_neox_layers_0_mlp_dense_h_to_4h_weight2[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 7)]; + vstore8((vload8(0, NT_matmul_rf_local + 0) + ((convert_float8(vload8(0, layer_norm50 + ((ax1_fused_u_fused_0 * 256) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_h_to_4h_weight2_local + 0))))), 0, NT_matmul_rf_local + 0); + } + for (int ax2_fused_2_0 = 0; ax2_fused_2_0 < 1; ++ax2_fused_2_0) { + NT_matmul_rf_local_1[0] = 0.000000e+00f; + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[0]); + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[1]); + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[2]); + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[3]); + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[4]); + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[5]); + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[6]); + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[7]); + } + barrier(CLK_LOCAL_MEM_FENCE); + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_rf_local_1[0]; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 16) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 8) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 4) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 2) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) == 0) { + compute[(((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0))))] = (convert_half(((((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) * 32)] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias2[(((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0))))]) * (5.000000e-01f + (erf(((((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) * 32)] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias2[(((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0))))]) * 7.071068e-01f)) * 5.000000e-01f))))); + } +} + +// Function: gpt_neox_q0f16_batch_prefill_paged_kv_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len) { + int tile_id[1]; + int batch_idx[1]; + int batch_rows[1]; + int batch_tiles[1]; + int kv_chunk_len[1]; + __local float m_smem[32]; + __local float d_smem[32]; + float O_local[32]; + __local half Q_smem[4096]; + __local half K_smem[2048]; + __local half V_smem[2048]; + float S_local[4]; + float m_prev[1]; + float m_new[1]; + float d_new[1]; + __local float m_prev_smem[32]; + tile_id[0] = (convert_int(get_group_id(0))); + batch_idx[0] = 0; + batch_rows[0] = (q_indptr[(q_indptr_elem_offset + 1)] - q_indptr[q_indptr_elem_offset]); + batch_tiles[0] = ((batch_rows[0] + 31) >> 5); + while (1) { + if (!(((batch_idx[0] < batch_size)))) { break; } + while (1) { + if (!((((bool)(batch_tiles[0] <= tile_id[0])) && ((bool)(batch_idx[0] < batch_size))))) { break; } + tile_id[0] = (tile_id[0] - batch_tiles[0]); + batch_idx[0] = (batch_idx[0] + 1); + if (batch_idx[0] < batch_size) { + int b_idx = batch_idx[0]; + batch_rows[0] = (q_indptr[((b_idx + q_indptr_elem_offset) + 1)] - q_indptr[(b_idx + q_indptr_elem_offset)]); + batch_tiles[0] = ((batch_rows[0] + 31) >> 5); + } + } + if ((batch_idx[0] < batch_size)) { + int b_idx_1 = batch_idx[0]; + int LH_start = (tile_id[0] * 32); + int q_indptr_val = q_indptr[(b_idx_1 + q_indptr_elem_offset)]; + int cur_page_indptr_begin = page_indptr[(b_idx_1 + page_indptr_elem_offset)]; + int cur_page_indptr_end = page_indptr[((b_idx_1 + page_indptr_elem_offset) + 1)]; + int condval; + if ((cur_page_indptr_begin != cur_page_indptr_end)) { + condval = ((((cur_page_indptr_end * 16) + length_info[(b_idx_1 + length_info_elem_offset)]) - (cur_page_indptr_begin * 16)) - 16); + } else { + condval = 0; + } + kv_chunk_len[0] = condval; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + m_smem[(convert_int(get_local_id(0)))] = -5.000000e+04f; + d_smem[(convert_int(get_local_id(0)))] = 1.000000e+00f; + } + for (int li_1 = 0; li_1 < 4; ++li_1) { + for (int lj_1 = 0; lj_1 < 8; ++lj_1) { + O_local[((li_1 * 8) + lj_1)] = 0.000000e+00f; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + for (int li_lj_fused_0 = 0; li_lj_fused_0 < 8; ++li_lj_fused_0) { + if (((((li_lj_fused_0 * 4) + q_indptr_val) + LH_start) + (convert_int(get_local_id(1)))) < q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)]) { + half4 condval_1; + if ((rotary_mode == 1)) { + float4 freq = (((float4)(((convert_float(q_rope_position[(((((li_lj_fused_0 * 4) + (convert_int(get_local_id(1)))) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[(((((li_lj_fused_0 * 4) + (convert_int(get_local_id(1)))) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[(((((li_lj_fused_0 * 4) + (convert_int(get_local_id(1)))) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[(((((li_lj_fused_0 * 4) + (convert_int(get_local_id(1)))) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale))) / pow(((float4)(rope_theta, rope_theta, rope_theta, rope_theta)), ((convert_float4(((int4)(((((convert_int(get_local_id(0))) & 15) * 8))+(2*0), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*1), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*2), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*3))))) * ((float4)(7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f))))); + half4 condval_2; + if (((convert_int(get_local_id(0))) < 16)) { + condval_2 = (vload4(0, q + (((((((li_lj_fused_0 * 8192) + ((convert_int(get_local_id(1))) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + 64)) * ((half4)((half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f))); + } else { + condval_2 = vload4(0, q + (((((((li_lj_fused_0 * 8192) + ((convert_int(get_local_id(1))) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) - 64)); + } + condval_1 = (convert_half4(((cos(freq) * (convert_float4(vload4(0, q + ((((((li_lj_fused_0 * 8192) + ((convert_int(get_local_id(1))) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)))))) + (sin(freq) * (convert_float4(condval_2)))))); + } else { + condval_1 = vload4(0, q + ((((((li_lj_fused_0 * 8192) + ((convert_int(get_local_id(1))) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); + } + vstore4(condval_1, 0, Q_smem + (((li_lj_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); + } else { + vstore4(((half4)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, Q_smem + (((li_lj_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); + } + } + barrier(CLK_LOCAL_MEM_FENCE); + for (int iterator = 0; iterator < ((kv_chunk_len[0] + 15) >> 4); ++iterator) { + barrier(CLK_LOCAL_MEM_FENCE); + for (int lz_ly_fused_0 = 0; lz_ly_fused_0 < 4; ++lz_ly_fused_0) { + if ((((iterator * 16) + (lz_ly_fused_0 * 4)) + (convert_int(get_local_id(1)))) < kv_chunk_len[0]) { + int page_no = page_values[((iterator + cur_page_indptr_begin) + page_values_elem_offset)]; + half4 condval_3; + if ((rotary_mode == 1)) { + float4 freq_1 = (((float4)(((convert_float(((((iterator * 16) + (lz_ly_fused_0 * 4)) + k_rope_pos_offset[(b_idx_1 + k_rope_pos_offset_elem_offset)]) + (convert_int(get_local_id(1)))))) * rope_scale), ((convert_float(((((iterator * 16) + (lz_ly_fused_0 * 4)) + k_rope_pos_offset[(b_idx_1 + k_rope_pos_offset_elem_offset)]) + (convert_int(get_local_id(1)))))) * rope_scale), ((convert_float(((((iterator * 16) + (lz_ly_fused_0 * 4)) + k_rope_pos_offset[(b_idx_1 + k_rope_pos_offset_elem_offset)]) + (convert_int(get_local_id(1)))))) * rope_scale), ((convert_float(((((iterator * 16) + (lz_ly_fused_0 * 4)) + k_rope_pos_offset[(b_idx_1 + k_rope_pos_offset_elem_offset)]) + (convert_int(get_local_id(1)))))) * rope_scale))) / pow(((float4)(rope_theta, rope_theta, rope_theta, rope_theta)), ((convert_float4(((int4)(((((convert_int(get_local_id(0))) & 15) * 8))+(2*0), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*1), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*2), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*3))))) * ((float4)(7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f))))); + int4 v_ = (int4)((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*0), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*1), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*2), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*3)); + half4 condval_4; + if (((convert_int(get_local_id(0))) < 16)) { + int4 v__1 = (int4)(((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 64))+(1*0), ((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 64))+(1*1), ((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 64))+(1*2), ((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 64))+(1*3)); + condval_4 = (((half4)(pages[v__1.s0],pages[v__1.s1],pages[v__1.s2],pages[v__1.s3])) * ((half4)((half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f))); + } else { + int4 v__2 = (int4)(((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) - 64))+(1*0), ((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) - 64))+(1*1), ((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) - 64))+(1*2), ((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) - 64))+(1*3)); + condval_4 = ((half4)(pages[v__2.s0],pages[v__2.s1],pages[v__2.s2],pages[v__2.s3])); + } + condval_3 = (convert_half4(((cos(freq_1) * (convert_float4(((half4)(pages[v_.s0],pages[v_.s1],pages[v_.s2],pages[v_.s3]))))) + (sin(freq_1) * (convert_float4(condval_4)))))); + } else { + int4 v__3 = (int4)((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*0), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*1), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*2), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*3)); + condval_3 = ((half4)(pages[v__3.s0],pages[v__3.s1],pages[v__3.s2],pages[v__3.s3])); + } + vstore4(condval_3, 0, K_smem + (((lz_ly_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); + } else { + vstore4(((half4)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, K_smem + (((lz_ly_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); + } + } + barrier(CLK_LOCAL_MEM_FENCE); + for (int lz_ly_fused_0_1 = 0; lz_ly_fused_0_1 < 4; ++lz_ly_fused_0_1) { + if ((((iterator * 16) + (lz_ly_fused_0_1 * 4)) + (convert_int(get_local_id(1)))) < kv_chunk_len[0]) { + int page_no_1 = page_values[((iterator + cur_page_indptr_begin) + page_values_elem_offset)]; + int4 v__4 = (int4)(((((((((page_no_1 * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0_1 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 32768))+(1*0), ((((((((page_no_1 * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0_1 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 32768))+(1*1), ((((((((page_no_1 * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0_1 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 32768))+(1*2), ((((((((page_no_1 * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0_1 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 32768))+(1*3)); + vstore4(((half4)(pages[v__4.s0],pages[v__4.s1],pages[v__4.s2],pages[v__4.s3])), 0, V_smem + (((lz_ly_fused_0_1 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); + } else { + vstore4(((half4)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, V_smem + (((lz_ly_fused_0_1 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); + } + } + barrier(CLK_LOCAL_MEM_FENCE); + for (int li_1_init = 0; li_1_init < 2; ++li_1_init) { + for (int lj_1_init = 0; lj_1_init < 2; ++lj_1_init) { + S_local[((li_1_init * 2) + lj_1_init)] = 0.000000e+00f; + } + } + for (int lk_0 = 0; lk_0 < 16; ++lk_0) { + for (int li_1_1 = 0; li_1_1 < 2; ++li_1_1) { + for (int lj_1_1 = 0; lj_1_1 < 2; ++lj_1_1) { + for (int lk_1 = 0; lk_1 < 8; ++lk_1) { + S_local[((li_1_1 * 2) + lj_1_1)] = (S_local[((li_1_1 * 2) + lj_1_1)] + ((((convert_float(Q_smem[((((((convert_int(get_local_id(1))) * 1024) + (((convert_int(get_local_id(0))) >> 3) * 256)) + (li_1_1 * 128)) + (lk_0 * 8)) + lk_1)])) * (convert_float(K_smem[((((((convert_int(get_local_id(0))) & 7) * 256) + (lj_1_1 * 128)) + (lk_0 * 8)) + lk_1)]))) * attn_score_scaling_factor) * 1.275174e-01f)); + } + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + for (int li_1_2 = 0; li_1_2 < 2; ++li_1_2) { + for (int lj_1_2 = 0; lj_1_2 < 2; ++lj_1_2) { + ((__local float*)K_smem)[((((((convert_int(get_local_id(1))) * 128) + (((convert_int(get_local_id(0))) >> 3) * 32)) + (li_1_2 * 16)) + (((convert_int(get_local_id(0))) & 7) * 2)) + lj_1_2)] = S_local[((li_1_2 * 2) + lj_1_2)]; + } + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + m_prev[0] = m_smem[(convert_int(get_local_id(0)))]; + m_new[0] = m_smem[(convert_int(get_local_id(0)))]; + for (int j = 0; j < 16; ++j) { + int cse_var_5 = ((iterator * 16) + j); + bool condval_5; + if ((0 < causal)) { + condval_5 = (cse_var_5 <= (((((((convert_int(get_local_id(1))) * 32) + kv_chunk_len[0]) + q_indptr[(b_idx_1 + q_indptr_elem_offset)]) + LH_start) + (convert_int(get_local_id(0)))) - q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)])); + } else { + condval_5 = (cse_var_5 < kv_chunk_len[0]); + } + if (condval_5) { + m_new[0] = max((float)m_new[0], (float)((__local float*)K_smem)[((((convert_int(get_local_id(1))) * 512) + ((convert_int(get_local_id(0))) * 16)) + j)]); + } + } + d_new[0] = (d_smem[(convert_int(get_local_id(0)))] * exp2((m_prev[0] - m_new[0]))); + } + for (int j_1 = 0; j_1 < 16; ++j_1) { + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + int cse_var_6 = ((iterator * 16) + j_1); + bool condval_6; + if ((0 < causal)) { + condval_6 = (cse_var_6 <= (((((((convert_int(get_local_id(1))) * 32) + kv_chunk_len[0]) + q_indptr[(b_idx_1 + q_indptr_elem_offset)]) + LH_start) + (convert_int(get_local_id(0)))) - q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)])); + } else { + condval_6 = (cse_var_6 < kv_chunk_len[0]); + } + if (condval_6) { + ((__local float*)K_smem)[((((convert_int(get_local_id(1))) * 512) + ((convert_int(get_local_id(0))) * 16)) + j_1)] = exp2((((__local float*)K_smem)[((((convert_int(get_local_id(1))) * 512) + ((convert_int(get_local_id(0))) * 16)) + j_1)] - m_new[0])); + } else { + ((__local float*)K_smem)[(((convert_int(get_local_id(0))) * 16) + j_1)] = exp2((-5.000000e+04f - m_new[0])); + } + } + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + for (int j_2 = 0; j_2 < 16; ++j_2) { + d_new[0] = (d_new[0] + ((__local float*)K_smem)[(((convert_int(get_local_id(0))) * 16) + j_2)]); + } + m_smem[(convert_int(get_local_id(0)))] = m_new[0]; + d_smem[(convert_int(get_local_id(0)))] = d_new[0]; + m_prev_smem[(convert_int(get_local_id(0)))] = m_prev[0]; + } + barrier(CLK_LOCAL_MEM_FENCE); + for (int li_1_init_1 = 0; li_1_init_1 < 4; ++li_1_init_1) { + for (int lj_1_init_1 = 0; lj_1_init_1 < 8; ++lj_1_init_1) { + O_local[((li_1_init_1 * 8) + lj_1_init_1)] = (O_local[((li_1_init_1 * 8) + lj_1_init_1)] * exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_init_1)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_init_1)]))); + } + } + for (int lk_0_1 = 0; lk_0_1 < 2; ++lk_0_1) { + for (int lk_1_1 = 0; lk_1_1 < 8; ++lk_1_1) { + for (int li_1_3 = 0; li_1_3 < 4; ++li_1_3) { + for (int lj_1_3 = 0; lj_1_3 < 8; ++lj_1_3) { + O_local[((li_1_3 * 8) + lj_1_3)] = (O_local[((li_1_3 * 8) + lj_1_3)] + (((__local float*)K_smem)[((((((convert_int(get_local_id(1))) * 128) + (((convert_int(get_local_id(0))) >> 4) * 64)) + (li_1_3 * 16)) + (lk_0_1 * 8)) + lk_1_1)] * (convert_float(V_smem[((((lk_0_1 * 1024) + (lk_1_1 * 128)) + (((convert_int(get_local_id(0))) & 15) * 8)) + lj_1_3)])))); + } + } + } + } + } + for (int li_1_4 = 0; li_1_4 < 4; ++li_1_4) { + for (int lj_1_4 = 0; lj_1_4 < 8; ++lj_1_4) { + int cur_L = ((((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + q_indptr[(b_idx_1 + q_indptr_elem_offset)]) + LH_start) + li_1_4); + if (cur_L < q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)]) { + output[((((cur_L * 2048) + ((convert_int(get_group_id(1))) * 128)) + (((convert_int(get_local_id(0))) & 15) * 8)) + lj_1_4)] = (convert_half((O_local[((li_1_4 * 8) + lj_1_4)] / d_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_4)]))); + } + } + } + if ((convert_int(get_local_id(1))) < 1) { + int cur_L_1 = (((((convert_int(get_local_id(1))) * 32) + q_indptr[(b_idx_1 + q_indptr_elem_offset)]) + LH_start) + (convert_int(get_local_id(0)))); + if (cur_L_1 < q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)]) { + lse[((cur_L_1 * 16) + (convert_int(get_group_id(1))))] = (m_smem[(convert_int(get_local_id(0)))] + log2(d_smem[(convert_int(get_local_id(0)))])); + } + } + tile_id[0] = (tile_id[0] + 16); + } + } +} + +// Function: gpt_neox_q0f16_take1_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size) { + T_take[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] = gpt_neox_embed_in_weight[(((input_ids[((convert_int(get_group_id(0))) >> 3)] * 2048) + (((convert_int(get_group_id(0))) & 7) * 256)) + (convert_int(get_local_id(0))))]; +} + +// Function: gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1 +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len) { + float NT_matmul_intermediate_pad_rf_local[64]; + float NT_matmul_intermediate_pad_rf_local_1[8]; + float NT_matmul_intermediate_pad_local[8]; + __local float red_buf0[64]; + for (int ax0_1_init = 0; ax0_1_init < 4; ++ax0_1_init) { + for (int ax1_fused_2_init = 0; ax1_fused_2_init < 2; ++ax1_fused_2_init) { + int8 v_ = (int8)((((ax0_1_init * 2) + ax1_fused_2_init))+(8*0), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*1), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*2), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*3), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*4), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*5), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*6), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*7)); + float8 v__1 = (float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f); + NT_matmul_intermediate_pad_rf_local[v_.s0] = v__1.s0; + NT_matmul_intermediate_pad_rf_local[v_.s1] = v__1.s1; + NT_matmul_intermediate_pad_rf_local[v_.s2] = v__1.s2; + NT_matmul_intermediate_pad_rf_local[v_.s3] = v__1.s3; + NT_matmul_intermediate_pad_rf_local[v_.s4] = v__1.s4; + NT_matmul_intermediate_pad_rf_local[v_.s5] = v__1.s5; + NT_matmul_intermediate_pad_rf_local[v_.s6] = v__1.s6; + NT_matmul_intermediate_pad_rf_local[v_.s7] = v__1.s7; + } + } + for (int ax2_fused_0 = 0; ax2_fused_0 < 32; ++ax2_fused_0) { + if (((convert_int(get_group_id(1))) * 4) < seq_len) { + int8 v__2 = (int8)((0)+(8*0), (0)+(8*1), (0)+(8*2), (0)+(8*3), (0)+(8*4), (0)+(8*5), (0)+(8*6), (0)+(8*7)); + float8 v__3 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__2.s0],NT_matmul_intermediate_pad_rf_local[v__2.s1],NT_matmul_intermediate_pad_rf_local[v__2.s2],NT_matmul_intermediate_pad_rf_local[v__2.s3],NT_matmul_intermediate_pad_rf_local[v__2.s4],NT_matmul_intermediate_pad_rf_local[v__2.s5],NT_matmul_intermediate_pad_rf_local[v__2.s6],NT_matmul_intermediate_pad_rf_local[v__2.s7])) + ((convert_float8(vload8(0, astype98 + ((((convert_int(get_group_id(1))) * 32768) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_weight3 + (((((convert_int(get_group_id(0))) * 32768) + ((convert_int(get_local_id(0))) * 16384)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))))); + NT_matmul_intermediate_pad_rf_local[v__2.s0] = v__3.s0; + NT_matmul_intermediate_pad_rf_local[v__2.s1] = v__3.s1; + NT_matmul_intermediate_pad_rf_local[v__2.s2] = v__3.s2; + NT_matmul_intermediate_pad_rf_local[v__2.s3] = v__3.s3; + NT_matmul_intermediate_pad_rf_local[v__2.s4] = v__3.s4; + NT_matmul_intermediate_pad_rf_local[v__2.s5] = v__3.s5; + NT_matmul_intermediate_pad_rf_local[v__2.s6] = v__3.s6; + NT_matmul_intermediate_pad_rf_local[v__2.s7] = v__3.s7; + int8 v__4 = (int8)((1)+(8*0), (1)+(8*1), (1)+(8*2), (1)+(8*3), (1)+(8*4), (1)+(8*5), (1)+(8*6), (1)+(8*7)); + float8 v__5 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__4.s0],NT_matmul_intermediate_pad_rf_local[v__4.s1],NT_matmul_intermediate_pad_rf_local[v__4.s2],NT_matmul_intermediate_pad_rf_local[v__4.s3],NT_matmul_intermediate_pad_rf_local[v__4.s4],NT_matmul_intermediate_pad_rf_local[v__4.s5],NT_matmul_intermediate_pad_rf_local[v__4.s6],NT_matmul_intermediate_pad_rf_local[v__4.s7])) + ((convert_float8(vload8(0, astype98 + ((((convert_int(get_group_id(1))) * 32768) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_weight3 + ((((((convert_int(get_group_id(0))) * 32768) + ((convert_int(get_local_id(0))) * 16384)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 8192))))); + NT_matmul_intermediate_pad_rf_local[v__4.s0] = v__5.s0; + NT_matmul_intermediate_pad_rf_local[v__4.s1] = v__5.s1; + NT_matmul_intermediate_pad_rf_local[v__4.s2] = v__5.s2; + NT_matmul_intermediate_pad_rf_local[v__4.s3] = v__5.s3; + NT_matmul_intermediate_pad_rf_local[v__4.s4] = v__5.s4; + NT_matmul_intermediate_pad_rf_local[v__4.s5] = v__5.s5; + NT_matmul_intermediate_pad_rf_local[v__4.s6] = v__5.s6; + NT_matmul_intermediate_pad_rf_local[v__4.s7] = v__5.s7; + } + if ((((convert_int(get_group_id(1))) * 4) + 1) < seq_len) { + int8 v__6 = (int8)((2)+(8*0), (2)+(8*1), (2)+(8*2), (2)+(8*3), (2)+(8*4), (2)+(8*5), (2)+(8*6), (2)+(8*7)); + float8 v__7 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__6.s0],NT_matmul_intermediate_pad_rf_local[v__6.s1],NT_matmul_intermediate_pad_rf_local[v__6.s2],NT_matmul_intermediate_pad_rf_local[v__6.s3],NT_matmul_intermediate_pad_rf_local[v__6.s4],NT_matmul_intermediate_pad_rf_local[v__6.s5],NT_matmul_intermediate_pad_rf_local[v__6.s6],NT_matmul_intermediate_pad_rf_local[v__6.s7])) + ((convert_float8(vload8(0, astype98 + (((((convert_int(get_group_id(1))) * 32768) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 8192)))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_weight3 + (((((convert_int(get_group_id(0))) * 32768) + ((convert_int(get_local_id(0))) * 16384)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))))); + NT_matmul_intermediate_pad_rf_local[v__6.s0] = v__7.s0; + NT_matmul_intermediate_pad_rf_local[v__6.s1] = v__7.s1; + NT_matmul_intermediate_pad_rf_local[v__6.s2] = v__7.s2; + NT_matmul_intermediate_pad_rf_local[v__6.s3] = v__7.s3; + NT_matmul_intermediate_pad_rf_local[v__6.s4] = v__7.s4; + NT_matmul_intermediate_pad_rf_local[v__6.s5] = v__7.s5; + NT_matmul_intermediate_pad_rf_local[v__6.s6] = v__7.s6; + NT_matmul_intermediate_pad_rf_local[v__6.s7] = v__7.s7; + int8 v__8 = (int8)((3)+(8*0), (3)+(8*1), (3)+(8*2), (3)+(8*3), (3)+(8*4), (3)+(8*5), (3)+(8*6), (3)+(8*7)); + float8 v__9 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__8.s0],NT_matmul_intermediate_pad_rf_local[v__8.s1],NT_matmul_intermediate_pad_rf_local[v__8.s2],NT_matmul_intermediate_pad_rf_local[v__8.s3],NT_matmul_intermediate_pad_rf_local[v__8.s4],NT_matmul_intermediate_pad_rf_local[v__8.s5],NT_matmul_intermediate_pad_rf_local[v__8.s6],NT_matmul_intermediate_pad_rf_local[v__8.s7])) + ((convert_float8(vload8(0, astype98 + (((((convert_int(get_group_id(1))) * 32768) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 8192)))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_weight3 + ((((((convert_int(get_group_id(0))) * 32768) + ((convert_int(get_local_id(0))) * 16384)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 8192))))); + NT_matmul_intermediate_pad_rf_local[v__8.s0] = v__9.s0; + NT_matmul_intermediate_pad_rf_local[v__8.s1] = v__9.s1; + NT_matmul_intermediate_pad_rf_local[v__8.s2] = v__9.s2; + NT_matmul_intermediate_pad_rf_local[v__8.s3] = v__9.s3; + NT_matmul_intermediate_pad_rf_local[v__8.s4] = v__9.s4; + NT_matmul_intermediate_pad_rf_local[v__8.s5] = v__9.s5; + NT_matmul_intermediate_pad_rf_local[v__8.s6] = v__9.s6; + NT_matmul_intermediate_pad_rf_local[v__8.s7] = v__9.s7; + } + if ((((convert_int(get_group_id(1))) * 4) + 2) < seq_len) { + int8 v__10 = (int8)((4)+(8*0), (4)+(8*1), (4)+(8*2), (4)+(8*3), (4)+(8*4), (4)+(8*5), (4)+(8*6), (4)+(8*7)); + float8 v__11 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__10.s0],NT_matmul_intermediate_pad_rf_local[v__10.s1],NT_matmul_intermediate_pad_rf_local[v__10.s2],NT_matmul_intermediate_pad_rf_local[v__10.s3],NT_matmul_intermediate_pad_rf_local[v__10.s4],NT_matmul_intermediate_pad_rf_local[v__10.s5],NT_matmul_intermediate_pad_rf_local[v__10.s6],NT_matmul_intermediate_pad_rf_local[v__10.s7])) + ((convert_float8(vload8(0, astype98 + (((((convert_int(get_group_id(1))) * 32768) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 16384)))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_weight3 + (((((convert_int(get_group_id(0))) * 32768) + ((convert_int(get_local_id(0))) * 16384)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))))); + NT_matmul_intermediate_pad_rf_local[v__10.s0] = v__11.s0; + NT_matmul_intermediate_pad_rf_local[v__10.s1] = v__11.s1; + NT_matmul_intermediate_pad_rf_local[v__10.s2] = v__11.s2; + NT_matmul_intermediate_pad_rf_local[v__10.s3] = v__11.s3; + NT_matmul_intermediate_pad_rf_local[v__10.s4] = v__11.s4; + NT_matmul_intermediate_pad_rf_local[v__10.s5] = v__11.s5; + NT_matmul_intermediate_pad_rf_local[v__10.s6] = v__11.s6; + NT_matmul_intermediate_pad_rf_local[v__10.s7] = v__11.s7; + int8 v__12 = (int8)((5)+(8*0), (5)+(8*1), (5)+(8*2), (5)+(8*3), (5)+(8*4), (5)+(8*5), (5)+(8*6), (5)+(8*7)); + float8 v__13 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__12.s0],NT_matmul_intermediate_pad_rf_local[v__12.s1],NT_matmul_intermediate_pad_rf_local[v__12.s2],NT_matmul_intermediate_pad_rf_local[v__12.s3],NT_matmul_intermediate_pad_rf_local[v__12.s4],NT_matmul_intermediate_pad_rf_local[v__12.s5],NT_matmul_intermediate_pad_rf_local[v__12.s6],NT_matmul_intermediate_pad_rf_local[v__12.s7])) + ((convert_float8(vload8(0, astype98 + (((((convert_int(get_group_id(1))) * 32768) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 16384)))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_weight3 + ((((((convert_int(get_group_id(0))) * 32768) + ((convert_int(get_local_id(0))) * 16384)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 8192))))); + NT_matmul_intermediate_pad_rf_local[v__12.s0] = v__13.s0; + NT_matmul_intermediate_pad_rf_local[v__12.s1] = v__13.s1; + NT_matmul_intermediate_pad_rf_local[v__12.s2] = v__13.s2; + NT_matmul_intermediate_pad_rf_local[v__12.s3] = v__13.s3; + NT_matmul_intermediate_pad_rf_local[v__12.s4] = v__13.s4; + NT_matmul_intermediate_pad_rf_local[v__12.s5] = v__13.s5; + NT_matmul_intermediate_pad_rf_local[v__12.s6] = v__13.s6; + NT_matmul_intermediate_pad_rf_local[v__12.s7] = v__13.s7; + } + if ((((convert_int(get_group_id(1))) * 4) + 3) < seq_len) { + int8 v__14 = (int8)((6)+(8*0), (6)+(8*1), (6)+(8*2), (6)+(8*3), (6)+(8*4), (6)+(8*5), (6)+(8*6), (6)+(8*7)); + float8 v__15 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__14.s0],NT_matmul_intermediate_pad_rf_local[v__14.s1],NT_matmul_intermediate_pad_rf_local[v__14.s2],NT_matmul_intermediate_pad_rf_local[v__14.s3],NT_matmul_intermediate_pad_rf_local[v__14.s4],NT_matmul_intermediate_pad_rf_local[v__14.s5],NT_matmul_intermediate_pad_rf_local[v__14.s6],NT_matmul_intermediate_pad_rf_local[v__14.s7])) + ((convert_float8(vload8(0, astype98 + (((((convert_int(get_group_id(1))) * 32768) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 24576)))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_weight3 + (((((convert_int(get_group_id(0))) * 32768) + ((convert_int(get_local_id(0))) * 16384)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))))); + NT_matmul_intermediate_pad_rf_local[v__14.s0] = v__15.s0; + NT_matmul_intermediate_pad_rf_local[v__14.s1] = v__15.s1; + NT_matmul_intermediate_pad_rf_local[v__14.s2] = v__15.s2; + NT_matmul_intermediate_pad_rf_local[v__14.s3] = v__15.s3; + NT_matmul_intermediate_pad_rf_local[v__14.s4] = v__15.s4; + NT_matmul_intermediate_pad_rf_local[v__14.s5] = v__15.s5; + NT_matmul_intermediate_pad_rf_local[v__14.s6] = v__15.s6; + NT_matmul_intermediate_pad_rf_local[v__14.s7] = v__15.s7; + int8 v__16 = (int8)((7)+(8*0), (7)+(8*1), (7)+(8*2), (7)+(8*3), (7)+(8*4), (7)+(8*5), (7)+(8*6), (7)+(8*7)); + float8 v__17 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__16.s0],NT_matmul_intermediate_pad_rf_local[v__16.s1],NT_matmul_intermediate_pad_rf_local[v__16.s2],NT_matmul_intermediate_pad_rf_local[v__16.s3],NT_matmul_intermediate_pad_rf_local[v__16.s4],NT_matmul_intermediate_pad_rf_local[v__16.s5],NT_matmul_intermediate_pad_rf_local[v__16.s6],NT_matmul_intermediate_pad_rf_local[v__16.s7])) + ((convert_float8(vload8(0, astype98 + (((((convert_int(get_group_id(1))) * 32768) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 24576)))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_weight3 + ((((((convert_int(get_group_id(0))) * 32768) + ((convert_int(get_local_id(0))) * 16384)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 8192))))); + NT_matmul_intermediate_pad_rf_local[v__16.s0] = v__17.s0; + NT_matmul_intermediate_pad_rf_local[v__16.s1] = v__17.s1; + NT_matmul_intermediate_pad_rf_local[v__16.s2] = v__17.s2; + NT_matmul_intermediate_pad_rf_local[v__16.s3] = v__17.s3; + NT_matmul_intermediate_pad_rf_local[v__16.s4] = v__17.s4; + NT_matmul_intermediate_pad_rf_local[v__16.s5] = v__17.s5; + NT_matmul_intermediate_pad_rf_local[v__16.s6] = v__17.s6; + NT_matmul_intermediate_pad_rf_local[v__16.s7] = v__17.s7; + } + } + for (int ax3_fused_2_0 = 0; ax3_fused_2_0 < 1; ++ax3_fused_2_0) { + for (int ax2 = 0; ax2 < 4; ++ax2) { + vstore2(((float2)(0.000000e+00f, 0.000000e+00f)), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + (ax2 * 2))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 8))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 16))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 24))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 32))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 40))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 48))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 56))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + } + } + for (int ax2_fused_2 = 0; ax2_fused_2 < 2; ++ax2_fused_2) { + for (int ax1 = 0; ax1 < 4; ++ax1) { + barrier(CLK_LOCAL_MEM_FENCE); + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_intermediate_pad_rf_local_1[((ax1 * 2) + ax2_fused_2)]; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 16) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 8) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 4) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 2) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + NT_matmul_intermediate_pad_local[((ax1 * 2) + ax2_fused_2)] = ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) * 32)]; + } + } + for (int ax0 = 0; ax0 < 4; ++ax0) { + for (int ax1_fused_2 = 0; ax1_fused_2 < 2; ++ax1_fused_2) { + if (((bool)(((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)(((bool)(((convert_int(get_group_id(1))) - ((seq_len + 3) >> 2)) < 0)) || ((bool)((convert_int(get_group_id(1))) == 0)))))) && ((bool)((((convert_int(get_group_id(1))) * 4) + ax0) < seq_len))) { + compute[((((((convert_int(get_group_id(1))) * 8192) + (ax0 * 2048)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] = (convert_half((NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias3[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]))); + } + } + } +} + +// Function: gpt_neox_q0f16_reshape7_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len) { + T_reshape[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] = reshape194[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]; +} + +// Function: gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size) { + half NT_matmul_intermediate_reindex_pad_local[16]; + for (int var = 0; var < 1; ++var) { + vstore8(((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, NT_matmul_intermediate_reindex_pad_local + 0); + vstore8(((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, NT_matmul_intermediate_reindex_pad_local + 8); + for (int ax3_0 = 0; ax3_0 < 128; ++ax3_0) { + for (int ax3_1 = 0; ax3_1 < 16; ++ax3_1) { + half condval; + if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { + condval = layer_norm195[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; + } else { + condval = (half)0.000000e+00f; + } + half condval_1; + if (((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) < vocab_size)) { + condval_1 = embed_out_weight4[(((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1)]; + } else { + condval_1 = (half)0.000000e+00f; + } + NT_matmul_intermediate_reindex_pad_local[0] = (NT_matmul_intermediate_reindex_pad_local[0] + (condval * condval_1)); + half condval_2; + if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { + condval_2 = layer_norm195[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; + } else { + condval_2 = (half)0.000000e+00f; + } + half condval_3; + if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 1) < vocab_size)) { + condval_3 = embed_out_weight4[((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1) + 2048)]; + } else { + condval_3 = (half)0.000000e+00f; + } + NT_matmul_intermediate_reindex_pad_local[1] = (NT_matmul_intermediate_reindex_pad_local[1] + (condval_2 * condval_3)); + half condval_4; + if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { + condval_4 = layer_norm195[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; + } else { + condval_4 = (half)0.000000e+00f; + } + half condval_5; + if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 2) < vocab_size)) { + condval_5 = embed_out_weight4[((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1) + 4096)]; + } else { + condval_5 = (half)0.000000e+00f; + } + NT_matmul_intermediate_reindex_pad_local[2] = (NT_matmul_intermediate_reindex_pad_local[2] + (condval_4 * condval_5)); + half condval_6; + if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { + condval_6 = layer_norm195[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; + } else { + condval_6 = (half)0.000000e+00f; + } + half condval_7; + if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 3) < vocab_size)) { + condval_7 = embed_out_weight4[((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1) + 6144)]; + } else { + condval_7 = (half)0.000000e+00f; + } + NT_matmul_intermediate_reindex_pad_local[3] = (NT_matmul_intermediate_reindex_pad_local[3] + (condval_6 * condval_7)); + half condval_8; + if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { + condval_8 = layer_norm195[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; + } else { + condval_8 = (half)0.000000e+00f; + } + half condval_9; + if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 4) < vocab_size)) { + condval_9 = embed_out_weight4[((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1) + 8192)]; + } else { + condval_9 = (half)0.000000e+00f; + } + NT_matmul_intermediate_reindex_pad_local[4] = (NT_matmul_intermediate_reindex_pad_local[4] + (condval_8 * condval_9)); + half condval_10; + if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { + condval_10 = layer_norm195[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; + } else { + condval_10 = (half)0.000000e+00f; + } + half condval_11; + if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 5) < vocab_size)) { + condval_11 = embed_out_weight4[((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1) + 10240)]; + } else { + condval_11 = (half)0.000000e+00f; + } + NT_matmul_intermediate_reindex_pad_local[5] = (NT_matmul_intermediate_reindex_pad_local[5] + (condval_10 * condval_11)); + half condval_12; + if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { + condval_12 = layer_norm195[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; + } else { + condval_12 = (half)0.000000e+00f; + } + half condval_13; + if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 6) < vocab_size)) { + condval_13 = embed_out_weight4[((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1) + 12288)]; + } else { + condval_13 = (half)0.000000e+00f; + } + NT_matmul_intermediate_reindex_pad_local[6] = (NT_matmul_intermediate_reindex_pad_local[6] + (condval_12 * condval_13)); + half condval_14; + if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { + condval_14 = layer_norm195[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; + } else { + condval_14 = (half)0.000000e+00f; + } + half condval_15; + if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 7) < vocab_size)) { + condval_15 = embed_out_weight4[((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1) + 14336)]; + } else { + condval_15 = (half)0.000000e+00f; + } + NT_matmul_intermediate_reindex_pad_local[7] = (NT_matmul_intermediate_reindex_pad_local[7] + (condval_14 * condval_15)); + half condval_16; + if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { + condval_16 = layer_norm195[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; + } else { + condval_16 = (half)0.000000e+00f; + } + half condval_17; + if (((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) < vocab_size)) { + condval_17 = embed_out_weight4[(((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1)]; + } else { + condval_17 = (half)0.000000e+00f; + } + NT_matmul_intermediate_reindex_pad_local[8] = (NT_matmul_intermediate_reindex_pad_local[8] + (condval_16 * condval_17)); + half condval_18; + if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { + condval_18 = layer_norm195[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; + } else { + condval_18 = (half)0.000000e+00f; + } + half condval_19; + if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 1) < vocab_size)) { + condval_19 = embed_out_weight4[((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1) + 2048)]; + } else { + condval_19 = (half)0.000000e+00f; + } + NT_matmul_intermediate_reindex_pad_local[9] = (NT_matmul_intermediate_reindex_pad_local[9] + (condval_18 * condval_19)); + half condval_20; + if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { + condval_20 = layer_norm195[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; + } else { + condval_20 = (half)0.000000e+00f; + } + half condval_21; + if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 2) < vocab_size)) { + condval_21 = embed_out_weight4[((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1) + 4096)]; + } else { + condval_21 = (half)0.000000e+00f; + } + NT_matmul_intermediate_reindex_pad_local[10] = (NT_matmul_intermediate_reindex_pad_local[10] + (condval_20 * condval_21)); + half condval_22; + if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { + condval_22 = layer_norm195[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; + } else { + condval_22 = (half)0.000000e+00f; + } + half condval_23; + if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 3) < vocab_size)) { + condval_23 = embed_out_weight4[((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1) + 6144)]; + } else { + condval_23 = (half)0.000000e+00f; + } + NT_matmul_intermediate_reindex_pad_local[11] = (NT_matmul_intermediate_reindex_pad_local[11] + (condval_22 * condval_23)); + half condval_24; + if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { + condval_24 = layer_norm195[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; + } else { + condval_24 = (half)0.000000e+00f; + } + half condval_25; + if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 4) < vocab_size)) { + condval_25 = embed_out_weight4[((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1) + 8192)]; + } else { + condval_25 = (half)0.000000e+00f; + } + NT_matmul_intermediate_reindex_pad_local[12] = (NT_matmul_intermediate_reindex_pad_local[12] + (condval_24 * condval_25)); + half condval_26; + if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { + condval_26 = layer_norm195[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; + } else { + condval_26 = (half)0.000000e+00f; + } + half condval_27; + if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 5) < vocab_size)) { + condval_27 = embed_out_weight4[((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1) + 10240)]; + } else { + condval_27 = (half)0.000000e+00f; + } + NT_matmul_intermediate_reindex_pad_local[13] = (NT_matmul_intermediate_reindex_pad_local[13] + (condval_26 * condval_27)); + half condval_28; + if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { + condval_28 = layer_norm195[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; + } else { + condval_28 = (half)0.000000e+00f; + } + half condval_29; + if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 6) < vocab_size)) { + condval_29 = embed_out_weight4[((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1) + 12288)]; + } else { + condval_29 = (half)0.000000e+00f; + } + NT_matmul_intermediate_reindex_pad_local[14] = (NT_matmul_intermediate_reindex_pad_local[14] + (condval_28 * condval_29)); + half condval_30; + if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { + condval_30 = layer_norm195[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; + } else { + condval_30 = (half)0.000000e+00f; + } + half condval_31; + if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 7) < vocab_size)) { + condval_31 = embed_out_weight4[((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1) + 14336)]; + } else { + condval_31 = (half)0.000000e+00f; + } + NT_matmul_intermediate_reindex_pad_local[15] = (NT_matmul_intermediate_reindex_pad_local[15] + (condval_30 * condval_31)); + } + } + if (((bool)((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) && ((bool)((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) < vocab_size))) { + compute[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) * vocab_size))] = (convert_float(NT_matmul_intermediate_reindex_pad_local[0])); + } + if (((bool)((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 1) < vocab_size))) { + compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) * vocab_size)) + 1)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[1])); + } + if (((bool)((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 2) < vocab_size))) { + compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) * vocab_size)) + 2)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[2])); + } + if (((bool)((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 3) < vocab_size))) { + compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) * vocab_size)) + 3)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[3])); + } + if (((bool)((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 4) < vocab_size))) { + compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) * vocab_size)) + 4)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[4])); + } + if (((bool)((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 5) < vocab_size))) { + compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) * vocab_size)) + 5)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[5])); + } + if (((bool)((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 6) < vocab_size))) { + compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) * vocab_size)) + 6)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[6])); + } + if (((bool)((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 7) < vocab_size))) { + compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) * vocab_size)) + 7)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[7])); + } + if (((bool)(((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) && ((bool)((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) < vocab_size))) { + compute[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) * vocab_size))] = (convert_float(NT_matmul_intermediate_reindex_pad_local[8])); + } + if (((bool)(((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 1) < vocab_size))) { + compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) * vocab_size)) + 1)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[9])); + } + if (((bool)(((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 2) < vocab_size))) { + compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) * vocab_size)) + 2)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[10])); + } + if (((bool)(((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 3) < vocab_size))) { + compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) * vocab_size)) + 3)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[11])); + } + if (((bool)(((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 4) < vocab_size))) { + compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) * vocab_size)) + 4)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[12])); + } + if (((bool)(((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 5) < vocab_size))) { + compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) * vocab_size)) + 5)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[13])); + } + if (((bool)(((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 6) < vocab_size))) { + compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) * vocab_size)) + 6)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[14])); + } + if (((bool)(((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 7) < vocab_size))) { + compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) * vocab_size)) + 7)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[15])); + } + } +} + +// Function: gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2 +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len) { + half8 NT_matmul_intermediate_reindex_pad_local[2]; + for (int var = 0; var < 1; ++var) { + NT_matmul_intermediate_reindex_pad_local[0] = ((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)); + NT_matmul_intermediate_reindex_pad_local[1] = ((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)); + for (int ax3_0 = 0; ax3_0 < 128; ++ax3_0) { + for (int ax3_1 = 0; ax3_1 < 16; ++ax3_1) { + half condval; + if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < seq_len)) { + condval = reshape195[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; + } else { + condval = (half)0.000000e+00f; + } + int8 v_ = (int8)(((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*0), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*1), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*2), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*3), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*4), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*5), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*6), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*7)); + NT_matmul_intermediate_reindex_pad_local[0] = (NT_matmul_intermediate_reindex_pad_local[0] + (((half8)(condval, condval, condval, condval, condval, condval, condval, condval)) * ((half8)(gpt_neox_layers_0_attention_dense_weight3[v_.s0],gpt_neox_layers_0_attention_dense_weight3[v_.s1],gpt_neox_layers_0_attention_dense_weight3[v_.s2],gpt_neox_layers_0_attention_dense_weight3[v_.s3],gpt_neox_layers_0_attention_dense_weight3[v_.s4],gpt_neox_layers_0_attention_dense_weight3[v_.s5],gpt_neox_layers_0_attention_dense_weight3[v_.s6],gpt_neox_layers_0_attention_dense_weight3[v_.s7])))); + half condval_1; + if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < seq_len)) { + condval_1 = reshape195[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; + } else { + condval_1 = (half)0.000000e+00f; + } + int8 v__1 = (int8)(((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*0), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*1), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*2), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*3), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*4), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*5), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*6), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*7)); + NT_matmul_intermediate_reindex_pad_local[1] = (NT_matmul_intermediate_reindex_pad_local[1] + (((half8)(condval_1, condval_1, condval_1, condval_1, condval_1, condval_1, condval_1, condval_1)) * ((half8)(gpt_neox_layers_0_attention_dense_weight3[v__1.s0],gpt_neox_layers_0_attention_dense_weight3[v__1.s1],gpt_neox_layers_0_attention_dense_weight3[v__1.s2],gpt_neox_layers_0_attention_dense_weight3[v__1.s3],gpt_neox_layers_0_attention_dense_weight3[v__1.s4],gpt_neox_layers_0_attention_dense_weight3[v__1.s5],gpt_neox_layers_0_attention_dense_weight3[v__1.s6],gpt_neox_layers_0_attention_dense_weight3[v__1.s7])))); + } + } + if ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < seq_len) { + vstore8(((vload8(0, astype99 + (((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8))) + (NT_matmul_intermediate_reindex_pad_local[0] + vload8(0, gpt_neox_layers_0_attention_dense_bias3 + (((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8))))) + vload8(0, input_embeds + (((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)))), 0, T_add + (((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8))); + } + if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < seq_len) { + vstore8(((vload8(0, astype99 + ((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 2048)) + (NT_matmul_intermediate_reindex_pad_local[1] + vload8(0, gpt_neox_layers_0_attention_dense_bias3 + (((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8))))) + vload8(0, input_embeds + ((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 2048))), 0, T_add + ((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 2048)); + } + } +} + +// Function: gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99) { + half NT_matmul_rf_local[8]; + half gpt_neox_layers_0_attention_dense_weight2_local[8]; + half NT_matmul_rf_local_1[1]; + __local half red_buf0[64]; + vstore8(((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, NT_matmul_rf_local + 0); + for (int ax1_fused_u_fused_0 = 0; ax1_fused_u_fused_0 < 8; ++ax1_fused_u_fused_0) { + gpt_neox_layers_0_attention_dense_weight2_local[0] = gpt_neox_layers_0_attention_dense_weight2[(((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))]; + gpt_neox_layers_0_attention_dense_weight2_local[1] = gpt_neox_layers_0_attention_dense_weight2[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 1)]; + gpt_neox_layers_0_attention_dense_weight2_local[2] = gpt_neox_layers_0_attention_dense_weight2[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2)]; + gpt_neox_layers_0_attention_dense_weight2_local[3] = gpt_neox_layers_0_attention_dense_weight2[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 3)]; + gpt_neox_layers_0_attention_dense_weight2_local[4] = gpt_neox_layers_0_attention_dense_weight2[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 4)]; + gpt_neox_layers_0_attention_dense_weight2_local[5] = gpt_neox_layers_0_attention_dense_weight2[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 5)]; + gpt_neox_layers_0_attention_dense_weight2_local[6] = gpt_neox_layers_0_attention_dense_weight2[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 6)]; + gpt_neox_layers_0_attention_dense_weight2_local[7] = gpt_neox_layers_0_attention_dense_weight2[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax1_fused_u_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 7)]; + vstore8((vload8(0, NT_matmul_rf_local + 0) + (vload8(0, reshape99 + ((ax1_fused_u_fused_0 * 256) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, gpt_neox_layers_0_attention_dense_weight2_local + 0))), 0, NT_matmul_rf_local + 0); + } + for (int ax2_fused_2_0 = 0; ax2_fused_2_0 < 1; ++ax2_fused_2_0) { + NT_matmul_rf_local_1[0] = (half)0.000000e+00f; + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[0]); + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[1]); + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[2]); + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[3]); + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[4]); + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[5]); + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[6]); + NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[7]); + } + barrier(CLK_LOCAL_MEM_FENCE); + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_rf_local_1[0]; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 16) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 8) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 4) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 2) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) == 0) { + T_add[(((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0))))] = ((astype50[(((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0))))] + (((volatile __local half*)red_buf0)[((convert_int(get_local_id(0))) * 32)] + gpt_neox_layers_0_attention_dense_bias2[(((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0))))])) + input_embed[(((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0))))]); + } +} + +// Function: gpt_neox_q0f16_layer_norm_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size) { + float in_thread_input_embeds_red_temp_v0_shared[1]; + float in_thread_input_embeds_red_temp_v1_shared[1]; + __local float red_buf0[256]; + __local float red_buf1[256]; + __local float input_embeds_red_temp_v0_shared[1]; + __local float input_embeds_red_temp_v1_shared[1]; + in_thread_input_embeds_red_temp_v0_shared[0] = 0.000000e+00f; + in_thread_input_embeds_red_temp_v1_shared[0] = 0.000000e+00f; + float v_input_embeds_red_temp_v0 = (in_thread_input_embeds_red_temp_v0_shared[0] + (convert_float(input_embeds[(((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0))))]))); + float v_input_embeds_red_temp_v1 = (in_thread_input_embeds_red_temp_v1_shared[0] + ((convert_float(input_embeds[(((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0))))])) * (convert_float(input_embeds[(((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0))))])))); + in_thread_input_embeds_red_temp_v0_shared[0] = v_input_embeds_red_temp_v0; + in_thread_input_embeds_red_temp_v1_shared[0] = v_input_embeds_red_temp_v1; + float v_input_embeds_red_temp_v0_1 = (in_thread_input_embeds_red_temp_v0_shared[0] + (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 256)]))); + float v_input_embeds_red_temp_v1_1 = (in_thread_input_embeds_red_temp_v1_shared[0] + ((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 256)])) * (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 256)])))); + in_thread_input_embeds_red_temp_v0_shared[0] = v_input_embeds_red_temp_v0_1; + in_thread_input_embeds_red_temp_v1_shared[0] = v_input_embeds_red_temp_v1_1; + float v_input_embeds_red_temp_v0_2 = (in_thread_input_embeds_red_temp_v0_shared[0] + (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 512)]))); + float v_input_embeds_red_temp_v1_2 = (in_thread_input_embeds_red_temp_v1_shared[0] + ((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 512)])) * (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 512)])))); + in_thread_input_embeds_red_temp_v0_shared[0] = v_input_embeds_red_temp_v0_2; + in_thread_input_embeds_red_temp_v1_shared[0] = v_input_embeds_red_temp_v1_2; + float v_input_embeds_red_temp_v0_3 = (in_thread_input_embeds_red_temp_v0_shared[0] + (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 768)]))); + float v_input_embeds_red_temp_v1_3 = (in_thread_input_embeds_red_temp_v1_shared[0] + ((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 768)])) * (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 768)])))); + in_thread_input_embeds_red_temp_v0_shared[0] = v_input_embeds_red_temp_v0_3; + in_thread_input_embeds_red_temp_v1_shared[0] = v_input_embeds_red_temp_v1_3; + float v_input_embeds_red_temp_v0_4 = (in_thread_input_embeds_red_temp_v0_shared[0] + (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1024)]))); + float v_input_embeds_red_temp_v1_4 = (in_thread_input_embeds_red_temp_v1_shared[0] + ((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1024)])) * (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1024)])))); + in_thread_input_embeds_red_temp_v0_shared[0] = v_input_embeds_red_temp_v0_4; + in_thread_input_embeds_red_temp_v1_shared[0] = v_input_embeds_red_temp_v1_4; + float v_input_embeds_red_temp_v0_5 = (in_thread_input_embeds_red_temp_v0_shared[0] + (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1280)]))); + float v_input_embeds_red_temp_v1_5 = (in_thread_input_embeds_red_temp_v1_shared[0] + ((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1280)])) * (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1280)])))); + in_thread_input_embeds_red_temp_v0_shared[0] = v_input_embeds_red_temp_v0_5; + in_thread_input_embeds_red_temp_v1_shared[0] = v_input_embeds_red_temp_v1_5; + float v_input_embeds_red_temp_v0_6 = (in_thread_input_embeds_red_temp_v0_shared[0] + (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1536)]))); + float v_input_embeds_red_temp_v1_6 = (in_thread_input_embeds_red_temp_v1_shared[0] + ((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1536)])) * (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1536)])))); + in_thread_input_embeds_red_temp_v0_shared[0] = v_input_embeds_red_temp_v0_6; + in_thread_input_embeds_red_temp_v1_shared[0] = v_input_embeds_red_temp_v1_6; + float v_input_embeds_red_temp_v0_7 = (in_thread_input_embeds_red_temp_v0_shared[0] + (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1792)]))); + float v_input_embeds_red_temp_v1_7 = (in_thread_input_embeds_red_temp_v1_shared[0] + ((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1792)])) * (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1792)])))); + in_thread_input_embeds_red_temp_v0_shared[0] = v_input_embeds_red_temp_v0_7; + in_thread_input_embeds_red_temp_v1_shared[0] = v_input_embeds_red_temp_v1_7; + barrier(CLK_LOCAL_MEM_FENCE); + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = in_thread_input_embeds_red_temp_v0_shared[0]; + ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = in_thread_input_embeds_red_temp_v1_shared[0]; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 128) { + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 128)]); + ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 128)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 64) { + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 64)]); + ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 64)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 32) { + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 32)]); + ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 32)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 16) { + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 16)]); + ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 16)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 8) { + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 8)]); + ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 8)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 4) { + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 4)]); + ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 4)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 2) { + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 2)]); + ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 2)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 1) { + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 1)]); + ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 1)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) == 0) { + input_embeds_red_temp_v0_shared[0] = ((volatile __local float*)red_buf0)[0]; + input_embeds_red_temp_v1_shared[0] = ((volatile __local float*)red_buf1)[0]; + } + barrier(CLK_LOCAL_MEM_FENCE); + T_layer_norm[(((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0))))] = (((convert_half((((convert_float(input_embeds[(((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0))))])) - (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embeds_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embeds_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight4[(convert_int(get_local_id(0)))]) + gpt_neox_layers_0_input_layernorm_bias4[(convert_int(get_local_id(0)))]); + T_layer_norm[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 256)] = (((convert_half((((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 256)])) - (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embeds_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embeds_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight4[((convert_int(get_local_id(0))) + 256)]) + gpt_neox_layers_0_input_layernorm_bias4[((convert_int(get_local_id(0))) + 256)]); + T_layer_norm[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 512)] = (((convert_half((((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 512)])) - (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embeds_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embeds_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight4[((convert_int(get_local_id(0))) + 512)]) + gpt_neox_layers_0_input_layernorm_bias4[((convert_int(get_local_id(0))) + 512)]); + T_layer_norm[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 768)] = (((convert_half((((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 768)])) - (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embeds_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embeds_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight4[((convert_int(get_local_id(0))) + 768)]) + gpt_neox_layers_0_input_layernorm_bias4[((convert_int(get_local_id(0))) + 768)]); + T_layer_norm[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1024)] = (((convert_half((((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1024)])) - (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embeds_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embeds_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight4[((convert_int(get_local_id(0))) + 1024)]) + gpt_neox_layers_0_input_layernorm_bias4[((convert_int(get_local_id(0))) + 1024)]); + T_layer_norm[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1280)] = (((convert_half((((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1280)])) - (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embeds_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embeds_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight4[((convert_int(get_local_id(0))) + 1280)]) + gpt_neox_layers_0_input_layernorm_bias4[((convert_int(get_local_id(0))) + 1280)]); + T_layer_norm[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1536)] = (((convert_half((((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1536)])) - (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embeds_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embeds_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight4[((convert_int(get_local_id(0))) + 1536)]) + gpt_neox_layers_0_input_layernorm_bias4[((convert_int(get_local_id(0))) + 1536)]); + T_layer_norm[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1792)] = (((convert_half((((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1792)])) - (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embeds_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embeds_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight4[((convert_int(get_local_id(0))) + 1792)]) + gpt_neox_layers_0_input_layernorm_bias4[((convert_int(get_local_id(0))) + 1792)]); +} + +// Function: gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len) { + half NT_matmul_intermediate_pad_rf_local[32]; + half NT_matmul_intermediate_pad_rf_local_1[4]; + half NT_matmul_intermediate_pad_local[4]; + __local half red_buf0[64]; + for (int ax0_1_init = 0; ax0_1_init < 2; ++ax0_1_init) { + for (int ax1_fused_2_init = 0; ax1_fused_2_init < 2; ++ax1_fused_2_init) { + int8 v_ = (int8)((((ax0_1_init * 2) + ax1_fused_2_init))+(4*0), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*1), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*2), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*3), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*4), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*5), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*6), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*7)); + half8 v__1 = (half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f); + NT_matmul_intermediate_pad_rf_local[v_.s0] = v__1.s0; + NT_matmul_intermediate_pad_rf_local[v_.s1] = v__1.s1; + NT_matmul_intermediate_pad_rf_local[v_.s2] = v__1.s2; + NT_matmul_intermediate_pad_rf_local[v_.s3] = v__1.s3; + NT_matmul_intermediate_pad_rf_local[v_.s4] = v__1.s4; + NT_matmul_intermediate_pad_rf_local[v_.s5] = v__1.s5; + NT_matmul_intermediate_pad_rf_local[v_.s6] = v__1.s6; + NT_matmul_intermediate_pad_rf_local[v_.s7] = v__1.s7; + } + } + for (int ax2_fused_0 = 0; ax2_fused_0 < 8; ++ax2_fused_0) { + if (((convert_int(get_group_id(1))) * 2) < seq_len) { + int8 v__2 = (int8)((0)+(4*0), (0)+(4*1), (0)+(4*2), (0)+(4*3), (0)+(4*4), (0)+(4*5), (0)+(4*6), (0)+(4*7)); + half8 v__3 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__2.s0],NT_matmul_intermediate_pad_rf_local[v__2.s1],NT_matmul_intermediate_pad_rf_local[v__2.s2],NT_matmul_intermediate_pad_rf_local[v__2.s3],NT_matmul_intermediate_pad_rf_local[v__2.s4],NT_matmul_intermediate_pad_rf_local[v__2.s5],NT_matmul_intermediate_pad_rf_local[v__2.s6],NT_matmul_intermediate_pad_rf_local[v__2.s7])) + (vload8(0, reshape195 + ((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, gpt_neox_layers_0_attention_dense_weight3 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))); + NT_matmul_intermediate_pad_rf_local[v__2.s0] = v__3.s0; + NT_matmul_intermediate_pad_rf_local[v__2.s1] = v__3.s1; + NT_matmul_intermediate_pad_rf_local[v__2.s2] = v__3.s2; + NT_matmul_intermediate_pad_rf_local[v__2.s3] = v__3.s3; + NT_matmul_intermediate_pad_rf_local[v__2.s4] = v__3.s4; + NT_matmul_intermediate_pad_rf_local[v__2.s5] = v__3.s5; + NT_matmul_intermediate_pad_rf_local[v__2.s6] = v__3.s6; + NT_matmul_intermediate_pad_rf_local[v__2.s7] = v__3.s7; + int8 v__4 = (int8)((1)+(4*0), (1)+(4*1), (1)+(4*2), (1)+(4*3), (1)+(4*4), (1)+(4*5), (1)+(4*6), (1)+(4*7)); + half8 v__5 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__4.s0],NT_matmul_intermediate_pad_rf_local[v__4.s1],NT_matmul_intermediate_pad_rf_local[v__4.s2],NT_matmul_intermediate_pad_rf_local[v__4.s3],NT_matmul_intermediate_pad_rf_local[v__4.s4],NT_matmul_intermediate_pad_rf_local[v__4.s5],NT_matmul_intermediate_pad_rf_local[v__4.s6],NT_matmul_intermediate_pad_rf_local[v__4.s7])) + (vload8(0, reshape195 + ((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, gpt_neox_layers_0_attention_dense_weight3 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))); + NT_matmul_intermediate_pad_rf_local[v__4.s0] = v__5.s0; + NT_matmul_intermediate_pad_rf_local[v__4.s1] = v__5.s1; + NT_matmul_intermediate_pad_rf_local[v__4.s2] = v__5.s2; + NT_matmul_intermediate_pad_rf_local[v__4.s3] = v__5.s3; + NT_matmul_intermediate_pad_rf_local[v__4.s4] = v__5.s4; + NT_matmul_intermediate_pad_rf_local[v__4.s5] = v__5.s5; + NT_matmul_intermediate_pad_rf_local[v__4.s6] = v__5.s6; + NT_matmul_intermediate_pad_rf_local[v__4.s7] = v__5.s7; + } + if ((((convert_int(get_group_id(1))) * 2) + 1) < seq_len) { + int8 v__6 = (int8)((2)+(4*0), (2)+(4*1), (2)+(4*2), (2)+(4*3), (2)+(4*4), (2)+(4*5), (2)+(4*6), (2)+(4*7)); + half8 v__7 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__6.s0],NT_matmul_intermediate_pad_rf_local[v__6.s1],NT_matmul_intermediate_pad_rf_local[v__6.s2],NT_matmul_intermediate_pad_rf_local[v__6.s3],NT_matmul_intermediate_pad_rf_local[v__6.s4],NT_matmul_intermediate_pad_rf_local[v__6.s5],NT_matmul_intermediate_pad_rf_local[v__6.s6],NT_matmul_intermediate_pad_rf_local[v__6.s7])) + (vload8(0, reshape195 + (((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)) * vload8(0, gpt_neox_layers_0_attention_dense_weight3 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))); + NT_matmul_intermediate_pad_rf_local[v__6.s0] = v__7.s0; + NT_matmul_intermediate_pad_rf_local[v__6.s1] = v__7.s1; + NT_matmul_intermediate_pad_rf_local[v__6.s2] = v__7.s2; + NT_matmul_intermediate_pad_rf_local[v__6.s3] = v__7.s3; + NT_matmul_intermediate_pad_rf_local[v__6.s4] = v__7.s4; + NT_matmul_intermediate_pad_rf_local[v__6.s5] = v__7.s5; + NT_matmul_intermediate_pad_rf_local[v__6.s6] = v__7.s6; + NT_matmul_intermediate_pad_rf_local[v__6.s7] = v__7.s7; + int8 v__8 = (int8)((3)+(4*0), (3)+(4*1), (3)+(4*2), (3)+(4*3), (3)+(4*4), (3)+(4*5), (3)+(4*6), (3)+(4*7)); + half8 v__9 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__8.s0],NT_matmul_intermediate_pad_rf_local[v__8.s1],NT_matmul_intermediate_pad_rf_local[v__8.s2],NT_matmul_intermediate_pad_rf_local[v__8.s3],NT_matmul_intermediate_pad_rf_local[v__8.s4],NT_matmul_intermediate_pad_rf_local[v__8.s5],NT_matmul_intermediate_pad_rf_local[v__8.s6],NT_matmul_intermediate_pad_rf_local[v__8.s7])) + (vload8(0, reshape195 + (((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)) * vload8(0, gpt_neox_layers_0_attention_dense_weight3 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))); + NT_matmul_intermediate_pad_rf_local[v__8.s0] = v__9.s0; + NT_matmul_intermediate_pad_rf_local[v__8.s1] = v__9.s1; + NT_matmul_intermediate_pad_rf_local[v__8.s2] = v__9.s2; + NT_matmul_intermediate_pad_rf_local[v__8.s3] = v__9.s3; + NT_matmul_intermediate_pad_rf_local[v__8.s4] = v__9.s4; + NT_matmul_intermediate_pad_rf_local[v__8.s5] = v__9.s5; + NT_matmul_intermediate_pad_rf_local[v__8.s6] = v__9.s6; + NT_matmul_intermediate_pad_rf_local[v__8.s7] = v__9.s7; + } + } + for (int ax3_fused_2_0 = 0; ax3_fused_2_0 < 1; ++ax3_fused_2_0) { + for (int ax2 = 0; ax2 < 2; ++ax2) { + vstore2(((half2)((half)0.000000e+00f, (half)0.000000e+00f)), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + (ax2 * 2))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 4))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 8))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 12))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 16))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 20))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 24))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 28))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + } + } + for (int ax2_fused_2 = 0; ax2_fused_2 < 2; ++ax2_fused_2) { + for (int ax1 = 0; ax1 < 2; ++ax1) { + barrier(CLK_LOCAL_MEM_FENCE); + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_intermediate_pad_rf_local_1[((ax1 * 2) + ax2_fused_2)]; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 16) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 8) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 4) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 2) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + NT_matmul_intermediate_pad_local[((ax1 * 2) + ax2_fused_2)] = ((volatile __local half*)red_buf0)[((convert_int(get_local_id(0))) * 32)]; + } + } + for (int ax0 = 0; ax0 < 2; ++ax0) { + for (int ax1_fused_2 = 0; ax1_fused_2 < 2; ++ax1_fused_2) { + if (((bool)(((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)(((bool)(((convert_int(get_group_id(1))) - ((seq_len + 1) >> 1)) < 0)) || ((bool)((convert_int(get_group_id(1))) == 0)))))) && ((bool)((((convert_int(get_group_id(1))) * 2) + ax0) < seq_len))) { + T_add[((((((convert_int(get_group_id(1))) * 4096) + (ax0 * 2048)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] = ((astype99[((((((convert_int(get_group_id(1))) * 4096) + (ax0 * 2048)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] + (NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_attention_dense_bias3[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)])) + input_embeds[((((((convert_int(get_group_id(1))) * 4096) + (ax0 * 2048)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]); + } + } + } +} + +// Function: gpt_neox_q0f16_chunk_lse_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size) { + float in_thread_temp_max_shared[1]; + __local float red_buf0[256]; + __local float temp_max_shared[1]; + float in_thread_temp_sum_shared[1]; + __local float red_buf0_1[256]; + __local float temp_sum_shared[1]; + in_thread_temp_max_shared[0] = -3.402823e+38f; + float condval; + if ((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) < vocab_size)) { + float condval_1; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_1 = (A[(((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0))))] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_1 = A[(((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0))))]; + } + condval = condval_1; + } else { + condval = -3.402823e+38f; + } + in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)condval); + float condval_2; + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 256) < vocab_size)) { + float condval_3; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_3 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 256)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_3 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 256)]; + } + condval_2 = condval_3; + } else { + condval_2 = -3.402823e+38f; + } + in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)condval_2); + float condval_4; + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 512) < vocab_size)) { + float condval_5; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_5 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 512)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_5 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 512)]; + } + condval_4 = condval_5; + } else { + condval_4 = -3.402823e+38f; + } + in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)condval_4); + float condval_6; + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 768) < vocab_size)) { + float condval_7; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_7 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 768)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_7 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 768)]; + } + condval_6 = condval_7; + } else { + condval_6 = -3.402823e+38f; + } + in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)condval_6); + float condval_8; + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 1024) < vocab_size)) { + float condval_9; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_9 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1024)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_9 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1024)]; + } + condval_8 = condval_9; + } else { + condval_8 = -3.402823e+38f; + } + in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)condval_8); + float condval_10; + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 1280) < vocab_size)) { + float condval_11; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_11 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1280)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_11 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1280)]; + } + condval_10 = condval_11; + } else { + condval_10 = -3.402823e+38f; + } + in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)condval_10); + float condval_12; + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 1536) < vocab_size)) { + float condval_13; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_13 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1536)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_13 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1536)]; + } + condval_12 = condval_13; + } else { + condval_12 = -3.402823e+38f; + } + in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)condval_12); + float condval_14; + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 1792) < vocab_size)) { + float condval_15; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_15 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1792)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_15 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1792)]; + } + condval_14 = condval_15; + } else { + condval_14 = -3.402823e+38f; + } + in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)condval_14); + float condval_16; + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 2048) < vocab_size)) { + float condval_17; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_17 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2048)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_17 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2048)]; + } + condval_16 = condval_17; + } else { + condval_16 = -3.402823e+38f; + } + in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)condval_16); + float condval_18; + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 2304) < vocab_size)) { + float condval_19; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_19 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2304)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_19 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2304)]; + } + condval_18 = condval_19; + } else { + condval_18 = -3.402823e+38f; + } + in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)condval_18); + float condval_20; + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 2560) < vocab_size)) { + float condval_21; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_21 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2560)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_21 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2560)]; + } + condval_20 = condval_21; + } else { + condval_20 = -3.402823e+38f; + } + in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)condval_20); + float condval_22; + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 2816) < vocab_size)) { + float condval_23; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_23 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2816)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_23 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2816)]; + } + condval_22 = condval_23; + } else { + condval_22 = -3.402823e+38f; + } + in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)condval_22); + float condval_24; + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 3072) < vocab_size)) { + float condval_25; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_25 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3072)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_25 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3072)]; + } + condval_24 = condval_25; + } else { + condval_24 = -3.402823e+38f; + } + in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)condval_24); + float condval_26; + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 3328) < vocab_size)) { + float condval_27; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_27 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3328)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_27 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3328)]; + } + condval_26 = condval_27; + } else { + condval_26 = -3.402823e+38f; + } + in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)condval_26); + float condval_28; + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 3584) < vocab_size)) { + float condval_29; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_29 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3584)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_29 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3584)]; + } + condval_28 = condval_29; + } else { + condval_28 = -3.402823e+38f; + } + in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)condval_28); + float condval_30; + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 3840) < vocab_size)) { + float condval_31; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_31 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3840)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_31 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3840)]; + } + condval_30 = condval_31; + } else { + condval_30 = -3.402823e+38f; + } + in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)condval_30); + barrier(CLK_LOCAL_MEM_FENCE); + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = in_thread_temp_max_shared[0]; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 128) { + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = max((float)((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))], (float)((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 128)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 64) { + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = max((float)((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))], (float)((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 64)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 32) { + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = max((float)((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))], (float)((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 32)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 16) { + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = max((float)((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))], (float)((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 16)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 8) { + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = max((float)((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))], (float)((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 8)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 4) { + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = max((float)((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))], (float)((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 4)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 2) { + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = max((float)((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))], (float)((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 2)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 1) { + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = max((float)((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))], (float)((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 1)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) == 0) { + temp_max_shared[0] = ((volatile __local float*)red_buf0)[0]; + } + in_thread_temp_sum_shared[0] = 0.000000e+00f; + barrier(CLK_LOCAL_MEM_FENCE); + float condval_32; + if ((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) < vocab_size)) { + float condval_33; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + float condval_34; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_34 = (A[(((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0))))] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_34 = A[(((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0))))]; + } + condval_33 = exp((condval_34 - temp_max_shared[0])); + } else { + float condval_35; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_35 = (A[(((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0))))] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_35 = A[(((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0))))]; + } + condval_33 = (convert_float((condval_35 == temp_max_shared[0]))); + } + condval_32 = condval_33; + } else { + condval_32 = 0.000000e+00f; + } + in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval_32); + float condval_36; + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 256) < vocab_size)) { + float condval_37; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + float condval_38; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_38 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 256)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_38 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 256)]; + } + condval_37 = exp((condval_38 - temp_max_shared[0])); + } else { + float condval_39; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_39 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 256)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_39 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 256)]; + } + condval_37 = (convert_float((condval_39 == temp_max_shared[0]))); + } + condval_36 = condval_37; + } else { + condval_36 = 0.000000e+00f; + } + in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval_36); + float condval_40; + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 512) < vocab_size)) { + float condval_41; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + float condval_42; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_42 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 512)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_42 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 512)]; + } + condval_41 = exp((condval_42 - temp_max_shared[0])); + } else { + float condval_43; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_43 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 512)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_43 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 512)]; + } + condval_41 = (convert_float((condval_43 == temp_max_shared[0]))); + } + condval_40 = condval_41; + } else { + condval_40 = 0.000000e+00f; + } + in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval_40); + float condval_44; + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 768) < vocab_size)) { + float condval_45; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + float condval_46; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_46 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 768)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_46 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 768)]; + } + condval_45 = exp((condval_46 - temp_max_shared[0])); + } else { + float condval_47; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_47 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 768)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_47 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 768)]; + } + condval_45 = (convert_float((condval_47 == temp_max_shared[0]))); + } + condval_44 = condval_45; + } else { + condval_44 = 0.000000e+00f; + } + in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval_44); + float condval_48; + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 1024) < vocab_size)) { + float condval_49; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + float condval_50; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_50 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1024)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_50 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1024)]; + } + condval_49 = exp((condval_50 - temp_max_shared[0])); + } else { + float condval_51; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_51 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1024)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_51 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1024)]; + } + condval_49 = (convert_float((condval_51 == temp_max_shared[0]))); + } + condval_48 = condval_49; + } else { + condval_48 = 0.000000e+00f; + } + in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval_48); + float condval_52; + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 1280) < vocab_size)) { + float condval_53; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + float condval_54; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_54 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1280)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_54 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1280)]; + } + condval_53 = exp((condval_54 - temp_max_shared[0])); + } else { + float condval_55; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_55 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1280)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_55 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1280)]; + } + condval_53 = (convert_float((condval_55 == temp_max_shared[0]))); + } + condval_52 = condval_53; + } else { + condval_52 = 0.000000e+00f; + } + in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval_52); + float condval_56; + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 1536) < vocab_size)) { + float condval_57; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + float condval_58; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_58 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1536)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_58 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1536)]; + } + condval_57 = exp((condval_58 - temp_max_shared[0])); + } else { + float condval_59; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_59 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1536)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_59 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1536)]; + } + condval_57 = (convert_float((condval_59 == temp_max_shared[0]))); + } + condval_56 = condval_57; + } else { + condval_56 = 0.000000e+00f; + } + in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval_56); + float condval_60; + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 1792) < vocab_size)) { + float condval_61; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + float condval_62; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_62 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1792)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_62 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1792)]; + } + condval_61 = exp((condval_62 - temp_max_shared[0])); + } else { + float condval_63; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_63 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1792)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_63 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1792)]; + } + condval_61 = (convert_float((condval_63 == temp_max_shared[0]))); + } + condval_60 = condval_61; + } else { + condval_60 = 0.000000e+00f; + } + in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval_60); + float condval_64; + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 2048) < vocab_size)) { + float condval_65; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + float condval_66; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_66 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2048)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_66 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2048)]; + } + condval_65 = exp((condval_66 - temp_max_shared[0])); + } else { + float condval_67; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_67 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2048)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_67 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2048)]; + } + condval_65 = (convert_float((condval_67 == temp_max_shared[0]))); + } + condval_64 = condval_65; + } else { + condval_64 = 0.000000e+00f; + } + in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval_64); + float condval_68; + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 2304) < vocab_size)) { + float condval_69; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + float condval_70; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_70 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2304)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_70 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2304)]; + } + condval_69 = exp((condval_70 - temp_max_shared[0])); + } else { + float condval_71; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_71 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2304)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_71 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2304)]; + } + condval_69 = (convert_float((condval_71 == temp_max_shared[0]))); + } + condval_68 = condval_69; + } else { + condval_68 = 0.000000e+00f; + } + in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval_68); + float condval_72; + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 2560) < vocab_size)) { + float condval_73; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + float condval_74; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_74 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2560)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_74 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2560)]; + } + condval_73 = exp((condval_74 - temp_max_shared[0])); + } else { + float condval_75; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_75 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2560)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_75 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2560)]; + } + condval_73 = (convert_float((condval_75 == temp_max_shared[0]))); + } + condval_72 = condval_73; + } else { + condval_72 = 0.000000e+00f; + } + in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval_72); + float condval_76; + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 2816) < vocab_size)) { + float condval_77; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + float condval_78; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_78 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2816)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_78 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2816)]; + } + condval_77 = exp((condval_78 - temp_max_shared[0])); + } else { + float condval_79; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_79 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2816)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_79 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2816)]; + } + condval_77 = (convert_float((condval_79 == temp_max_shared[0]))); + } + condval_76 = condval_77; + } else { + condval_76 = 0.000000e+00f; + } + in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval_76); + float condval_80; + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 3072) < vocab_size)) { + float condval_81; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + float condval_82; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_82 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3072)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_82 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3072)]; + } + condval_81 = exp((condval_82 - temp_max_shared[0])); + } else { + float condval_83; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_83 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3072)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_83 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3072)]; + } + condval_81 = (convert_float((condval_83 == temp_max_shared[0]))); + } + condval_80 = condval_81; + } else { + condval_80 = 0.000000e+00f; + } + in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval_80); + float condval_84; + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 3328) < vocab_size)) { + float condval_85; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + float condval_86; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_86 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3328)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_86 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3328)]; + } + condval_85 = exp((condval_86 - temp_max_shared[0])); + } else { + float condval_87; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_87 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3328)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_87 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3328)]; + } + condval_85 = (convert_float((condval_87 == temp_max_shared[0]))); + } + condval_84 = condval_85; + } else { + condval_84 = 0.000000e+00f; + } + in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval_84); + float condval_88; + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 3584) < vocab_size)) { + float condval_89; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + float condval_90; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_90 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3584)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_90 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3584)]; + } + condval_89 = exp((condval_90 - temp_max_shared[0])); + } else { + float condval_91; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_91 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3584)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_91 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3584)]; + } + condval_89 = (convert_float((condval_91 == temp_max_shared[0]))); + } + condval_88 = condval_89; + } else { + condval_88 = 0.000000e+00f; + } + in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval_88); + float condval_92; + if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 3840) < vocab_size)) { + float condval_93; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + float condval_94; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_94 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3840)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_94 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3840)]; + } + condval_93 = exp((condval_94 - temp_max_shared[0])); + } else { + float condval_95; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_95 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3840)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); + } else { + condval_95 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3840)]; + } + condval_93 = (convert_float((condval_95 == temp_max_shared[0]))); + } + condval_92 = condval_93; + } else { + condval_92 = 0.000000e+00f; + } + in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval_92); + barrier(CLK_LOCAL_MEM_FENCE); + ((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] = in_thread_temp_sum_shared[0]; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 128) { + ((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0_1)[((convert_int(get_local_id(0))) + 128)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 64) { + ((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0_1)[((convert_int(get_local_id(0))) + 64)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 32) { + ((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0_1)[((convert_int(get_local_id(0))) + 32)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 16) { + ((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0_1)[((convert_int(get_local_id(0))) + 16)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 8) { + ((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0_1)[((convert_int(get_local_id(0))) + 8)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 4) { + ((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0_1)[((convert_int(get_local_id(0))) + 4)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 2) { + ((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0_1)[((convert_int(get_local_id(0))) + 2)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 1) { + ((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0_1)[((convert_int(get_local_id(0))) + 1)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) == 0) { + temp_sum_shared[0] = ((volatile __local float*)red_buf0_1)[0]; + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 1) { + float condval_96; + if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { + condval_96 = log(temp_sum_shared[0]); + } else { + condval_96 = temp_sum_shared[0]; + } + chunked_sum[(convert_int(get_group_id(0)))] = condval_96; + chunked_max[(convert_int(get_group_id(0)))] = temp_max_shared[0]; + } +} + +// Function: gpt_neox_q0f16_fused_NT_matmul_add_kernel_1 +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size) { + half NT_matmul_intermediate_pad_rf_local[64]; + half NT_matmul_intermediate_pad_rf_local_1[8]; + half NT_matmul_intermediate_pad_local[8]; + __local half red_buf0[64]; + for (int ax0_1_init = 0; ax0_1_init < 4; ++ax0_1_init) { + for (int ax1_fused_2_init = 0; ax1_fused_2_init < 2; ++ax1_fused_2_init) { + int8 v_ = (int8)((((ax0_1_init * 2) + ax1_fused_2_init))+(8*0), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*1), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*2), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*3), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*4), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*5), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*6), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*7)); + half8 v__1 = (half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f); + NT_matmul_intermediate_pad_rf_local[v_.s0] = v__1.s0; + NT_matmul_intermediate_pad_rf_local[v_.s1] = v__1.s1; + NT_matmul_intermediate_pad_rf_local[v_.s2] = v__1.s2; + NT_matmul_intermediate_pad_rf_local[v_.s3] = v__1.s3; + NT_matmul_intermediate_pad_rf_local[v_.s4] = v__1.s4; + NT_matmul_intermediate_pad_rf_local[v_.s5] = v__1.s5; + NT_matmul_intermediate_pad_rf_local[v_.s6] = v__1.s6; + NT_matmul_intermediate_pad_rf_local[v_.s7] = v__1.s7; + } + } + for (int ax2_fused_0 = 0; ax2_fused_0 < 8; ++ax2_fused_0) { + if (((convert_int(get_group_id(1))) * 4) < batch_size) { + int8 v__2 = (int8)((0)+(8*0), (0)+(8*1), (0)+(8*2), (0)+(8*3), (0)+(8*4), (0)+(8*5), (0)+(8*6), (0)+(8*7)); + half8 v__3 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__2.s0],NT_matmul_intermediate_pad_rf_local[v__2.s1],NT_matmul_intermediate_pad_rf_local[v__2.s2],NT_matmul_intermediate_pad_rf_local[v__2.s3],NT_matmul_intermediate_pad_rf_local[v__2.s4],NT_matmul_intermediate_pad_rf_local[v__2.s5],NT_matmul_intermediate_pad_rf_local[v__2.s6],NT_matmul_intermediate_pad_rf_local[v__2.s7])) + (vload8(0, layer_norm147 + ((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, gpt_neox_layers_0_attention_query_key_value_weight4 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))); + NT_matmul_intermediate_pad_rf_local[v__2.s0] = v__3.s0; + NT_matmul_intermediate_pad_rf_local[v__2.s1] = v__3.s1; + NT_matmul_intermediate_pad_rf_local[v__2.s2] = v__3.s2; + NT_matmul_intermediate_pad_rf_local[v__2.s3] = v__3.s3; + NT_matmul_intermediate_pad_rf_local[v__2.s4] = v__3.s4; + NT_matmul_intermediate_pad_rf_local[v__2.s5] = v__3.s5; + NT_matmul_intermediate_pad_rf_local[v__2.s6] = v__3.s6; + NT_matmul_intermediate_pad_rf_local[v__2.s7] = v__3.s7; + int8 v__4 = (int8)((1)+(8*0), (1)+(8*1), (1)+(8*2), (1)+(8*3), (1)+(8*4), (1)+(8*5), (1)+(8*6), (1)+(8*7)); + half8 v__5 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__4.s0],NT_matmul_intermediate_pad_rf_local[v__4.s1],NT_matmul_intermediate_pad_rf_local[v__4.s2],NT_matmul_intermediate_pad_rf_local[v__4.s3],NT_matmul_intermediate_pad_rf_local[v__4.s4],NT_matmul_intermediate_pad_rf_local[v__4.s5],NT_matmul_intermediate_pad_rf_local[v__4.s6],NT_matmul_intermediate_pad_rf_local[v__4.s7])) + (vload8(0, layer_norm147 + ((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, gpt_neox_layers_0_attention_query_key_value_weight4 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))); + NT_matmul_intermediate_pad_rf_local[v__4.s0] = v__5.s0; + NT_matmul_intermediate_pad_rf_local[v__4.s1] = v__5.s1; + NT_matmul_intermediate_pad_rf_local[v__4.s2] = v__5.s2; + NT_matmul_intermediate_pad_rf_local[v__4.s3] = v__5.s3; + NT_matmul_intermediate_pad_rf_local[v__4.s4] = v__5.s4; + NT_matmul_intermediate_pad_rf_local[v__4.s5] = v__5.s5; + NT_matmul_intermediate_pad_rf_local[v__4.s6] = v__5.s6; + NT_matmul_intermediate_pad_rf_local[v__4.s7] = v__5.s7; + } + if ((((convert_int(get_group_id(1))) * 4) + 1) < batch_size) { + int8 v__6 = (int8)((2)+(8*0), (2)+(8*1), (2)+(8*2), (2)+(8*3), (2)+(8*4), (2)+(8*5), (2)+(8*6), (2)+(8*7)); + half8 v__7 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__6.s0],NT_matmul_intermediate_pad_rf_local[v__6.s1],NT_matmul_intermediate_pad_rf_local[v__6.s2],NT_matmul_intermediate_pad_rf_local[v__6.s3],NT_matmul_intermediate_pad_rf_local[v__6.s4],NT_matmul_intermediate_pad_rf_local[v__6.s5],NT_matmul_intermediate_pad_rf_local[v__6.s6],NT_matmul_intermediate_pad_rf_local[v__6.s7])) + (vload8(0, layer_norm147 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)) * vload8(0, gpt_neox_layers_0_attention_query_key_value_weight4 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))); + NT_matmul_intermediate_pad_rf_local[v__6.s0] = v__7.s0; + NT_matmul_intermediate_pad_rf_local[v__6.s1] = v__7.s1; + NT_matmul_intermediate_pad_rf_local[v__6.s2] = v__7.s2; + NT_matmul_intermediate_pad_rf_local[v__6.s3] = v__7.s3; + NT_matmul_intermediate_pad_rf_local[v__6.s4] = v__7.s4; + NT_matmul_intermediate_pad_rf_local[v__6.s5] = v__7.s5; + NT_matmul_intermediate_pad_rf_local[v__6.s6] = v__7.s6; + NT_matmul_intermediate_pad_rf_local[v__6.s7] = v__7.s7; + int8 v__8 = (int8)((3)+(8*0), (3)+(8*1), (3)+(8*2), (3)+(8*3), (3)+(8*4), (3)+(8*5), (3)+(8*6), (3)+(8*7)); + half8 v__9 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__8.s0],NT_matmul_intermediate_pad_rf_local[v__8.s1],NT_matmul_intermediate_pad_rf_local[v__8.s2],NT_matmul_intermediate_pad_rf_local[v__8.s3],NT_matmul_intermediate_pad_rf_local[v__8.s4],NT_matmul_intermediate_pad_rf_local[v__8.s5],NT_matmul_intermediate_pad_rf_local[v__8.s6],NT_matmul_intermediate_pad_rf_local[v__8.s7])) + (vload8(0, layer_norm147 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)) * vload8(0, gpt_neox_layers_0_attention_query_key_value_weight4 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))); + NT_matmul_intermediate_pad_rf_local[v__8.s0] = v__9.s0; + NT_matmul_intermediate_pad_rf_local[v__8.s1] = v__9.s1; + NT_matmul_intermediate_pad_rf_local[v__8.s2] = v__9.s2; + NT_matmul_intermediate_pad_rf_local[v__8.s3] = v__9.s3; + NT_matmul_intermediate_pad_rf_local[v__8.s4] = v__9.s4; + NT_matmul_intermediate_pad_rf_local[v__8.s5] = v__9.s5; + NT_matmul_intermediate_pad_rf_local[v__8.s6] = v__9.s6; + NT_matmul_intermediate_pad_rf_local[v__8.s7] = v__9.s7; + } + if ((((convert_int(get_group_id(1))) * 4) + 2) < batch_size) { + int8 v__10 = (int8)((4)+(8*0), (4)+(8*1), (4)+(8*2), (4)+(8*3), (4)+(8*4), (4)+(8*5), (4)+(8*6), (4)+(8*7)); + half8 v__11 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__10.s0],NT_matmul_intermediate_pad_rf_local[v__10.s1],NT_matmul_intermediate_pad_rf_local[v__10.s2],NT_matmul_intermediate_pad_rf_local[v__10.s3],NT_matmul_intermediate_pad_rf_local[v__10.s4],NT_matmul_intermediate_pad_rf_local[v__10.s5],NT_matmul_intermediate_pad_rf_local[v__10.s6],NT_matmul_intermediate_pad_rf_local[v__10.s7])) + (vload8(0, layer_norm147 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 4096)) * vload8(0, gpt_neox_layers_0_attention_query_key_value_weight4 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))); + NT_matmul_intermediate_pad_rf_local[v__10.s0] = v__11.s0; + NT_matmul_intermediate_pad_rf_local[v__10.s1] = v__11.s1; + NT_matmul_intermediate_pad_rf_local[v__10.s2] = v__11.s2; + NT_matmul_intermediate_pad_rf_local[v__10.s3] = v__11.s3; + NT_matmul_intermediate_pad_rf_local[v__10.s4] = v__11.s4; + NT_matmul_intermediate_pad_rf_local[v__10.s5] = v__11.s5; + NT_matmul_intermediate_pad_rf_local[v__10.s6] = v__11.s6; + NT_matmul_intermediate_pad_rf_local[v__10.s7] = v__11.s7; + int8 v__12 = (int8)((5)+(8*0), (5)+(8*1), (5)+(8*2), (5)+(8*3), (5)+(8*4), (5)+(8*5), (5)+(8*6), (5)+(8*7)); + half8 v__13 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__12.s0],NT_matmul_intermediate_pad_rf_local[v__12.s1],NT_matmul_intermediate_pad_rf_local[v__12.s2],NT_matmul_intermediate_pad_rf_local[v__12.s3],NT_matmul_intermediate_pad_rf_local[v__12.s4],NT_matmul_intermediate_pad_rf_local[v__12.s5],NT_matmul_intermediate_pad_rf_local[v__12.s6],NT_matmul_intermediate_pad_rf_local[v__12.s7])) + (vload8(0, layer_norm147 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 4096)) * vload8(0, gpt_neox_layers_0_attention_query_key_value_weight4 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))); + NT_matmul_intermediate_pad_rf_local[v__12.s0] = v__13.s0; + NT_matmul_intermediate_pad_rf_local[v__12.s1] = v__13.s1; + NT_matmul_intermediate_pad_rf_local[v__12.s2] = v__13.s2; + NT_matmul_intermediate_pad_rf_local[v__12.s3] = v__13.s3; + NT_matmul_intermediate_pad_rf_local[v__12.s4] = v__13.s4; + NT_matmul_intermediate_pad_rf_local[v__12.s5] = v__13.s5; + NT_matmul_intermediate_pad_rf_local[v__12.s6] = v__13.s6; + NT_matmul_intermediate_pad_rf_local[v__12.s7] = v__13.s7; + } + if ((((convert_int(get_group_id(1))) * 4) + 3) < batch_size) { + int8 v__14 = (int8)((6)+(8*0), (6)+(8*1), (6)+(8*2), (6)+(8*3), (6)+(8*4), (6)+(8*5), (6)+(8*6), (6)+(8*7)); + half8 v__15 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__14.s0],NT_matmul_intermediate_pad_rf_local[v__14.s1],NT_matmul_intermediate_pad_rf_local[v__14.s2],NT_matmul_intermediate_pad_rf_local[v__14.s3],NT_matmul_intermediate_pad_rf_local[v__14.s4],NT_matmul_intermediate_pad_rf_local[v__14.s5],NT_matmul_intermediate_pad_rf_local[v__14.s6],NT_matmul_intermediate_pad_rf_local[v__14.s7])) + (vload8(0, layer_norm147 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 6144)) * vload8(0, gpt_neox_layers_0_attention_query_key_value_weight4 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))); + NT_matmul_intermediate_pad_rf_local[v__14.s0] = v__15.s0; + NT_matmul_intermediate_pad_rf_local[v__14.s1] = v__15.s1; + NT_matmul_intermediate_pad_rf_local[v__14.s2] = v__15.s2; + NT_matmul_intermediate_pad_rf_local[v__14.s3] = v__15.s3; + NT_matmul_intermediate_pad_rf_local[v__14.s4] = v__15.s4; + NT_matmul_intermediate_pad_rf_local[v__14.s5] = v__15.s5; + NT_matmul_intermediate_pad_rf_local[v__14.s6] = v__15.s6; + NT_matmul_intermediate_pad_rf_local[v__14.s7] = v__15.s7; + int8 v__16 = (int8)((7)+(8*0), (7)+(8*1), (7)+(8*2), (7)+(8*3), (7)+(8*4), (7)+(8*5), (7)+(8*6), (7)+(8*7)); + half8 v__17 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__16.s0],NT_matmul_intermediate_pad_rf_local[v__16.s1],NT_matmul_intermediate_pad_rf_local[v__16.s2],NT_matmul_intermediate_pad_rf_local[v__16.s3],NT_matmul_intermediate_pad_rf_local[v__16.s4],NT_matmul_intermediate_pad_rf_local[v__16.s5],NT_matmul_intermediate_pad_rf_local[v__16.s6],NT_matmul_intermediate_pad_rf_local[v__16.s7])) + (vload8(0, layer_norm147 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 6144)) * vload8(0, gpt_neox_layers_0_attention_query_key_value_weight4 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))); + NT_matmul_intermediate_pad_rf_local[v__16.s0] = v__17.s0; + NT_matmul_intermediate_pad_rf_local[v__16.s1] = v__17.s1; + NT_matmul_intermediate_pad_rf_local[v__16.s2] = v__17.s2; + NT_matmul_intermediate_pad_rf_local[v__16.s3] = v__17.s3; + NT_matmul_intermediate_pad_rf_local[v__16.s4] = v__17.s4; + NT_matmul_intermediate_pad_rf_local[v__16.s5] = v__17.s5; + NT_matmul_intermediate_pad_rf_local[v__16.s6] = v__17.s6; + NT_matmul_intermediate_pad_rf_local[v__16.s7] = v__17.s7; + } + } + for (int ax3_fused_2_0 = 0; ax3_fused_2_0 < 1; ++ax3_fused_2_0) { + for (int ax2 = 0; ax2 < 4; ++ax2) { + vstore2(((half2)((half)0.000000e+00f, (half)0.000000e+00f)), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + (ax2 * 2))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 8))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 16))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 24))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 32))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 40))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 48))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 56))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + } + } + for (int ax2_fused_2 = 0; ax2_fused_2 < 2; ++ax2_fused_2) { + for (int ax1 = 0; ax1 < 4; ++ax1) { + barrier(CLK_LOCAL_MEM_FENCE); + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_intermediate_pad_rf_local_1[((ax1 * 2) + ax2_fused_2)]; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 16) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 8) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 4) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 2) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + NT_matmul_intermediate_pad_local[((ax1 * 2) + ax2_fused_2)] = ((volatile __local half*)red_buf0)[((convert_int(get_local_id(0))) * 32)]; + } + } + for (int ax0 = 0; ax0 < 4; ++ax0) { + for (int ax1_fused_2 = 0; ax1_fused_2 < 2; ++ax1_fused_2) { + if (((bool)(((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)(((bool)(((convert_int(get_group_id(1))) - ((batch_size + 3) >> 2)) < 0)) || ((bool)((convert_int(get_group_id(1))) == 0)))))) && ((bool)((((convert_int(get_group_id(1))) * 4) + ax0) < batch_size))) { + T_add[((((((convert_int(get_group_id(1))) * 24576) + (ax0 * 6144)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] = (NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_attention_query_key_value_bias4[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]); + } + } + } +} + +// Function: gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size) { + float NT_matmul_intermediate_pad_rf_local[32]; + float NT_matmul_intermediate_pad_rf_local_1[4]; + float NT_matmul_intermediate_pad_local[4]; + __local float red_buf0[64]; + for (int ax0_1_init = 0; ax0_1_init < 2; ++ax0_1_init) { + for (int ax1_fused_2_init = 0; ax1_fused_2_init < 2; ++ax1_fused_2_init) { + int8 v_ = (int8)((((ax0_1_init * 2) + ax1_fused_2_init))+(4*0), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*1), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*2), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*3), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*4), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*5), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*6), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*7)); + float8 v__1 = (float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f); + NT_matmul_intermediate_pad_rf_local[v_.s0] = v__1.s0; + NT_matmul_intermediate_pad_rf_local[v_.s1] = v__1.s1; + NT_matmul_intermediate_pad_rf_local[v_.s2] = v__1.s2; + NT_matmul_intermediate_pad_rf_local[v_.s3] = v__1.s3; + NT_matmul_intermediate_pad_rf_local[v_.s4] = v__1.s4; + NT_matmul_intermediate_pad_rf_local[v_.s5] = v__1.s5; + NT_matmul_intermediate_pad_rf_local[v_.s6] = v__1.s6; + NT_matmul_intermediate_pad_rf_local[v_.s7] = v__1.s7; + } + } + for (int ax2_fused_0 = 0; ax2_fused_0 < 8; ++ax2_fused_0) { + if (((convert_int(get_group_id(1))) * 2) < batch_size) { + int8 v__2 = (int8)((0)+(4*0), (0)+(4*1), (0)+(4*2), (0)+(4*3), (0)+(4*4), (0)+(4*5), (0)+(4*6), (0)+(4*7)); + float8 v__3 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__2.s0],NT_matmul_intermediate_pad_rf_local[v__2.s1],NT_matmul_intermediate_pad_rf_local[v__2.s2],NT_matmul_intermediate_pad_rf_local[v__2.s3],NT_matmul_intermediate_pad_rf_local[v__2.s4],NT_matmul_intermediate_pad_rf_local[v__2.s5],NT_matmul_intermediate_pad_rf_local[v__2.s6],NT_matmul_intermediate_pad_rf_local[v__2.s7])) + ((convert_float8(vload8(0, layer_norm148 + ((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_h_to_4h_weight4 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))))); + NT_matmul_intermediate_pad_rf_local[v__2.s0] = v__3.s0; + NT_matmul_intermediate_pad_rf_local[v__2.s1] = v__3.s1; + NT_matmul_intermediate_pad_rf_local[v__2.s2] = v__3.s2; + NT_matmul_intermediate_pad_rf_local[v__2.s3] = v__3.s3; + NT_matmul_intermediate_pad_rf_local[v__2.s4] = v__3.s4; + NT_matmul_intermediate_pad_rf_local[v__2.s5] = v__3.s5; + NT_matmul_intermediate_pad_rf_local[v__2.s6] = v__3.s6; + NT_matmul_intermediate_pad_rf_local[v__2.s7] = v__3.s7; + int8 v__4 = (int8)((1)+(4*0), (1)+(4*1), (1)+(4*2), (1)+(4*3), (1)+(4*4), (1)+(4*5), (1)+(4*6), (1)+(4*7)); + float8 v__5 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__4.s0],NT_matmul_intermediate_pad_rf_local[v__4.s1],NT_matmul_intermediate_pad_rf_local[v__4.s2],NT_matmul_intermediate_pad_rf_local[v__4.s3],NT_matmul_intermediate_pad_rf_local[v__4.s4],NT_matmul_intermediate_pad_rf_local[v__4.s5],NT_matmul_intermediate_pad_rf_local[v__4.s6],NT_matmul_intermediate_pad_rf_local[v__4.s7])) + ((convert_float8(vload8(0, layer_norm148 + ((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_h_to_4h_weight4 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))))); + NT_matmul_intermediate_pad_rf_local[v__4.s0] = v__5.s0; + NT_matmul_intermediate_pad_rf_local[v__4.s1] = v__5.s1; + NT_matmul_intermediate_pad_rf_local[v__4.s2] = v__5.s2; + NT_matmul_intermediate_pad_rf_local[v__4.s3] = v__5.s3; + NT_matmul_intermediate_pad_rf_local[v__4.s4] = v__5.s4; + NT_matmul_intermediate_pad_rf_local[v__4.s5] = v__5.s5; + NT_matmul_intermediate_pad_rf_local[v__4.s6] = v__5.s6; + NT_matmul_intermediate_pad_rf_local[v__4.s7] = v__5.s7; + } + if ((((convert_int(get_group_id(1))) * 2) + 1) < batch_size) { + int8 v__6 = (int8)((2)+(4*0), (2)+(4*1), (2)+(4*2), (2)+(4*3), (2)+(4*4), (2)+(4*5), (2)+(4*6), (2)+(4*7)); + float8 v__7 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__6.s0],NT_matmul_intermediate_pad_rf_local[v__6.s1],NT_matmul_intermediate_pad_rf_local[v__6.s2],NT_matmul_intermediate_pad_rf_local[v__6.s3],NT_matmul_intermediate_pad_rf_local[v__6.s4],NT_matmul_intermediate_pad_rf_local[v__6.s5],NT_matmul_intermediate_pad_rf_local[v__6.s6],NT_matmul_intermediate_pad_rf_local[v__6.s7])) + ((convert_float8(vload8(0, layer_norm148 + (((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_h_to_4h_weight4 + (((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))))); + NT_matmul_intermediate_pad_rf_local[v__6.s0] = v__7.s0; + NT_matmul_intermediate_pad_rf_local[v__6.s1] = v__7.s1; + NT_matmul_intermediate_pad_rf_local[v__6.s2] = v__7.s2; + NT_matmul_intermediate_pad_rf_local[v__6.s3] = v__7.s3; + NT_matmul_intermediate_pad_rf_local[v__6.s4] = v__7.s4; + NT_matmul_intermediate_pad_rf_local[v__6.s5] = v__7.s5; + NT_matmul_intermediate_pad_rf_local[v__6.s6] = v__7.s6; + NT_matmul_intermediate_pad_rf_local[v__6.s7] = v__7.s7; + int8 v__8 = (int8)((3)+(4*0), (3)+(4*1), (3)+(4*2), (3)+(4*3), (3)+(4*4), (3)+(4*5), (3)+(4*6), (3)+(4*7)); + float8 v__9 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__8.s0],NT_matmul_intermediate_pad_rf_local[v__8.s1],NT_matmul_intermediate_pad_rf_local[v__8.s2],NT_matmul_intermediate_pad_rf_local[v__8.s3],NT_matmul_intermediate_pad_rf_local[v__8.s4],NT_matmul_intermediate_pad_rf_local[v__8.s5],NT_matmul_intermediate_pad_rf_local[v__8.s6],NT_matmul_intermediate_pad_rf_local[v__8.s7])) + ((convert_float8(vload8(0, layer_norm148 + (((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_h_to_4h_weight4 + ((((((convert_int(get_group_id(0))) * 8192) + ((convert_int(get_local_id(0))) * 4096)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048))))); + NT_matmul_intermediate_pad_rf_local[v__8.s0] = v__9.s0; + NT_matmul_intermediate_pad_rf_local[v__8.s1] = v__9.s1; + NT_matmul_intermediate_pad_rf_local[v__8.s2] = v__9.s2; + NT_matmul_intermediate_pad_rf_local[v__8.s3] = v__9.s3; + NT_matmul_intermediate_pad_rf_local[v__8.s4] = v__9.s4; + NT_matmul_intermediate_pad_rf_local[v__8.s5] = v__9.s5; + NT_matmul_intermediate_pad_rf_local[v__8.s6] = v__9.s6; + NT_matmul_intermediate_pad_rf_local[v__8.s7] = v__9.s7; + } + } + for (int ax3_fused_2_0 = 0; ax3_fused_2_0 < 1; ++ax3_fused_2_0) { + for (int ax2 = 0; ax2 < 2; ++ax2) { + vstore2(((float2)(0.000000e+00f, 0.000000e+00f)), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + (ax2 * 2))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 4))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 8))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 12))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 16))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 20))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 24))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 28))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + } + } + for (int ax2_fused_2 = 0; ax2_fused_2 < 2; ++ax2_fused_2) { + for (int ax1 = 0; ax1 < 2; ++ax1) { + barrier(CLK_LOCAL_MEM_FENCE); + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_intermediate_pad_rf_local_1[((ax1 * 2) + ax2_fused_2)]; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 16) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 8) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 4) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 2) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + NT_matmul_intermediate_pad_local[((ax1 * 2) + ax2_fused_2)] = ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) * 32)]; + } + } + for (int ax0 = 0; ax0 < 2; ++ax0) { + for (int ax1_fused_2 = 0; ax1_fused_2 < 2; ++ax1_fused_2) { + if (((bool)(((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)(((bool)(((convert_int(get_group_id(1))) - ((batch_size + 1) >> 1)) < 0)) || ((bool)((convert_int(get_group_id(1))) == 0)))))) && ((bool)((((convert_int(get_group_id(1))) * 2) + ax0) < batch_size))) { + compute[((((((convert_int(get_group_id(1))) * 16384) + (ax0 * 8192)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] = (convert_half(((NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]) * 7.071068e-01f)) * 5.000000e-01f))))); + } + } + } +} + +// Function: gpt_neox_q0f16_reshape_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size) { + T_reshape[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] = add432[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]; +} + +// Function: gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size) { + float NT_matmul_intermediate_pad_rf_local[32]; + float NT_matmul_intermediate_pad_rf_local_1[4]; + float NT_matmul_intermediate_pad_local[4]; + __local float red_buf0[64]; + for (int ax0_1_init = 0; ax0_1_init < 2; ++ax0_1_init) { + for (int ax1_fused_2_init = 0; ax1_fused_2_init < 2; ++ax1_fused_2_init) { + int8 v_ = (int8)((((ax0_1_init * 2) + ax1_fused_2_init))+(4*0), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*1), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*2), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*3), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*4), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*5), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*6), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*7)); + float8 v__1 = (float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f); + NT_matmul_intermediate_pad_rf_local[v_.s0] = v__1.s0; + NT_matmul_intermediate_pad_rf_local[v_.s1] = v__1.s1; + NT_matmul_intermediate_pad_rf_local[v_.s2] = v__1.s2; + NT_matmul_intermediate_pad_rf_local[v_.s3] = v__1.s3; + NT_matmul_intermediate_pad_rf_local[v_.s4] = v__1.s4; + NT_matmul_intermediate_pad_rf_local[v_.s5] = v__1.s5; + NT_matmul_intermediate_pad_rf_local[v_.s6] = v__1.s6; + NT_matmul_intermediate_pad_rf_local[v_.s7] = v__1.s7; + } + } + for (int ax2_fused_0 = 0; ax2_fused_0 < 32; ++ax2_fused_0) { + if (((convert_int(get_group_id(1))) * 2) < batch_size) { + int8 v__2 = (int8)((0)+(4*0), (0)+(4*1), (0)+(4*2), (0)+(4*3), (0)+(4*4), (0)+(4*5), (0)+(4*6), (0)+(4*7)); + float8 v__3 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__2.s0],NT_matmul_intermediate_pad_rf_local[v__2.s1],NT_matmul_intermediate_pad_rf_local[v__2.s2],NT_matmul_intermediate_pad_rf_local[v__2.s3],NT_matmul_intermediate_pad_rf_local[v__2.s4],NT_matmul_intermediate_pad_rf_local[v__2.s5],NT_matmul_intermediate_pad_rf_local[v__2.s6],NT_matmul_intermediate_pad_rf_local[v__2.s7])) + ((convert_float8(vload8(0, astype147 + ((((convert_int(get_group_id(1))) * 16384) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_weight4 + (((((convert_int(get_group_id(0))) * 32768) + ((convert_int(get_local_id(0))) * 16384)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))))); + NT_matmul_intermediate_pad_rf_local[v__2.s0] = v__3.s0; + NT_matmul_intermediate_pad_rf_local[v__2.s1] = v__3.s1; + NT_matmul_intermediate_pad_rf_local[v__2.s2] = v__3.s2; + NT_matmul_intermediate_pad_rf_local[v__2.s3] = v__3.s3; + NT_matmul_intermediate_pad_rf_local[v__2.s4] = v__3.s4; + NT_matmul_intermediate_pad_rf_local[v__2.s5] = v__3.s5; + NT_matmul_intermediate_pad_rf_local[v__2.s6] = v__3.s6; + NT_matmul_intermediate_pad_rf_local[v__2.s7] = v__3.s7; + int8 v__4 = (int8)((1)+(4*0), (1)+(4*1), (1)+(4*2), (1)+(4*3), (1)+(4*4), (1)+(4*5), (1)+(4*6), (1)+(4*7)); + float8 v__5 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__4.s0],NT_matmul_intermediate_pad_rf_local[v__4.s1],NT_matmul_intermediate_pad_rf_local[v__4.s2],NT_matmul_intermediate_pad_rf_local[v__4.s3],NT_matmul_intermediate_pad_rf_local[v__4.s4],NT_matmul_intermediate_pad_rf_local[v__4.s5],NT_matmul_intermediate_pad_rf_local[v__4.s6],NT_matmul_intermediate_pad_rf_local[v__4.s7])) + ((convert_float8(vload8(0, astype147 + ((((convert_int(get_group_id(1))) * 16384) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_weight4 + ((((((convert_int(get_group_id(0))) * 32768) + ((convert_int(get_local_id(0))) * 16384)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 8192))))); + NT_matmul_intermediate_pad_rf_local[v__4.s0] = v__5.s0; + NT_matmul_intermediate_pad_rf_local[v__4.s1] = v__5.s1; + NT_matmul_intermediate_pad_rf_local[v__4.s2] = v__5.s2; + NT_matmul_intermediate_pad_rf_local[v__4.s3] = v__5.s3; + NT_matmul_intermediate_pad_rf_local[v__4.s4] = v__5.s4; + NT_matmul_intermediate_pad_rf_local[v__4.s5] = v__5.s5; + NT_matmul_intermediate_pad_rf_local[v__4.s6] = v__5.s6; + NT_matmul_intermediate_pad_rf_local[v__4.s7] = v__5.s7; + } + if ((((convert_int(get_group_id(1))) * 2) + 1) < batch_size) { + int8 v__6 = (int8)((2)+(4*0), (2)+(4*1), (2)+(4*2), (2)+(4*3), (2)+(4*4), (2)+(4*5), (2)+(4*6), (2)+(4*7)); + float8 v__7 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__6.s0],NT_matmul_intermediate_pad_rf_local[v__6.s1],NT_matmul_intermediate_pad_rf_local[v__6.s2],NT_matmul_intermediate_pad_rf_local[v__6.s3],NT_matmul_intermediate_pad_rf_local[v__6.s4],NT_matmul_intermediate_pad_rf_local[v__6.s5],NT_matmul_intermediate_pad_rf_local[v__6.s6],NT_matmul_intermediate_pad_rf_local[v__6.s7])) + ((convert_float8(vload8(0, astype147 + (((((convert_int(get_group_id(1))) * 16384) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 8192)))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_weight4 + (((((convert_int(get_group_id(0))) * 32768) + ((convert_int(get_local_id(0))) * 16384)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)))))); + NT_matmul_intermediate_pad_rf_local[v__6.s0] = v__7.s0; + NT_matmul_intermediate_pad_rf_local[v__6.s1] = v__7.s1; + NT_matmul_intermediate_pad_rf_local[v__6.s2] = v__7.s2; + NT_matmul_intermediate_pad_rf_local[v__6.s3] = v__7.s3; + NT_matmul_intermediate_pad_rf_local[v__6.s4] = v__7.s4; + NT_matmul_intermediate_pad_rf_local[v__6.s5] = v__7.s5; + NT_matmul_intermediate_pad_rf_local[v__6.s6] = v__7.s6; + NT_matmul_intermediate_pad_rf_local[v__6.s7] = v__7.s7; + int8 v__8 = (int8)((3)+(4*0), (3)+(4*1), (3)+(4*2), (3)+(4*3), (3)+(4*4), (3)+(4*5), (3)+(4*6), (3)+(4*7)); + float8 v__9 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__8.s0],NT_matmul_intermediate_pad_rf_local[v__8.s1],NT_matmul_intermediate_pad_rf_local[v__8.s2],NT_matmul_intermediate_pad_rf_local[v__8.s3],NT_matmul_intermediate_pad_rf_local[v__8.s4],NT_matmul_intermediate_pad_rf_local[v__8.s5],NT_matmul_intermediate_pad_rf_local[v__8.s6],NT_matmul_intermediate_pad_rf_local[v__8.s7])) + ((convert_float8(vload8(0, astype147 + (((((convert_int(get_group_id(1))) * 16384) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 8192)))) * (convert_float8(vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_weight4 + ((((((convert_int(get_group_id(0))) * 32768) + ((convert_int(get_local_id(0))) * 16384)) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 8192))))); + NT_matmul_intermediate_pad_rf_local[v__8.s0] = v__9.s0; + NT_matmul_intermediate_pad_rf_local[v__8.s1] = v__9.s1; + NT_matmul_intermediate_pad_rf_local[v__8.s2] = v__9.s2; + NT_matmul_intermediate_pad_rf_local[v__8.s3] = v__9.s3; + NT_matmul_intermediate_pad_rf_local[v__8.s4] = v__9.s4; + NT_matmul_intermediate_pad_rf_local[v__8.s5] = v__9.s5; + NT_matmul_intermediate_pad_rf_local[v__8.s6] = v__9.s6; + NT_matmul_intermediate_pad_rf_local[v__8.s7] = v__9.s7; + } + } + for (int ax3_fused_2_0 = 0; ax3_fused_2_0 < 1; ++ax3_fused_2_0) { + for (int ax2 = 0; ax2 < 2; ++ax2) { + vstore2(((float2)(0.000000e+00f, 0.000000e+00f)), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + (ax2 * 2))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 4))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 8))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 12))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 16))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 20))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 24))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 28))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); + } + } + for (int ax2_fused_2 = 0; ax2_fused_2 < 2; ++ax2_fused_2) { + for (int ax1 = 0; ax1 < 2; ++ax1) { + barrier(CLK_LOCAL_MEM_FENCE); + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_intermediate_pad_rf_local_1[((ax1 * 2) + ax2_fused_2)]; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 16) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 8) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 4) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 2) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(1))) < 1) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + NT_matmul_intermediate_pad_local[((ax1 * 2) + ax2_fused_2)] = ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) * 32)]; + } + } + for (int ax0 = 0; ax0 < 2; ++ax0) { + for (int ax1_fused_2 = 0; ax1_fused_2 < 2; ++ax1_fused_2) { + if (((bool)(((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)(((bool)(((convert_int(get_group_id(1))) - ((batch_size + 1) >> 1)) < 0)) || ((bool)((convert_int(get_group_id(1))) == 0)))))) && ((bool)((((convert_int(get_group_id(1))) * 2) + ax0) < batch_size))) { + compute[((((((convert_int(get_group_id(1))) * 4096) + (ax0 * 2048)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] = (convert_half((NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias4[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]))); + } + } + } +} + +// Function: gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode) { + int kv_chunk_len[1]; + float st_m[1]; + float st_d[1]; + float4 O_local[1]; + half4 Q_local[1]; + float m_prev[1]; + __local float4 O_allreduce[256]; + __local float md_allreduce[16]; + __local half4 K_smem[256]; + __local half4 V_smem[256]; + float QK_local[4]; + float S_reduce_local[1]; + __local float red_buf0[256]; + float S_local[1]; + half4 V_local[1]; + float d_prev[1]; + float other_m[1]; + float other_d[1]; + float4 other_o[1]; + float exp_mprev[1]; + int cur_page_indptr_begin = page_table_indptr[((convert_int(get_group_id(0))) + page_indptr_elem_offset)]; + int cur_page_indptr_end = page_table_indptr[(((convert_int(get_group_id(0))) + page_indptr_elem_offset) + 1)]; + int condval; + if ((cur_page_indptr_begin != cur_page_indptr_end)) { + condval = ((((((cur_page_indptr_end * 16) + length_info[((convert_int(get_group_id(0))) + length_info_elem_offset)]) + length_info[(((B * 2) + (convert_int(get_group_id(0)))) + length_info_elem_offset)]) - length_info[(((convert_int(get_group_id(0))) + B) + length_info_elem_offset)]) - (cur_page_indptr_begin * 16)) - 16); + } else { + condval = 0; + } + kv_chunk_len[0] = condval; + st_m[0] = -5.000000e+04f; + st_d[0] = 1.000000e+00f; + O_local[0] = ((float4)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)); + half4 condval_1; + if ((rotary_mode == 1)) { + float4 freq = (((float4)(((convert_float(q_rope_position[((convert_int(get_group_id(0))) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[((convert_int(get_group_id(0))) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[((convert_int(get_group_id(0))) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[((convert_int(get_group_id(0))) + q_rope_position_elem_offset)])) * rope_scale))) / pow(((float4)(rope_theta, rope_theta, rope_theta, rope_theta)), ((convert_float4(((int4)(((((convert_int(get_local_id(0))) & 15) * 8))+(2*0), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*1), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*2), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*3))))) * ((float4)(7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f))))); + half4 condval_2; + if (((convert_int(get_local_id(0))) < 16)) { + condval_2 = (vload4(0, Q + (((((convert_int(get_group_id(0))) * 2048) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + 64)) * ((half4)((half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f))); + } else { + condval_2 = vload4(0, Q + (((((convert_int(get_group_id(0))) * 2048) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) - 64)); + } + condval_1 = (convert_half4(((cos(freq) * (convert_float4(vload4(0, Q + ((((convert_int(get_group_id(0))) * 2048) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)))))) + (sin(freq) * (convert_float4(condval_2)))))); + } else { + condval_1 = vload4(0, Q + ((((convert_int(get_group_id(0))) * 2048) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); + } + Q_local[0] = condval_1; + for (int iterator = 0; iterator < ((kv_chunk_len[0] + 7) >> 3); ++iterator) { + if (((iterator * 8) + (convert_int(get_local_id(2)))) < kv_chunk_len[0]) { + int condval_3; + if ((((iterator * 8) + (convert_int(get_local_id(2)))) < length_info[(((B * 2) + (convert_int(get_group_id(0)))) + length_info_elem_offset)])) { + condval_3 = ((iterator * 8) + (convert_int(get_local_id(2)))); + } else { + condval_3 = ((((iterator * 8) + (convert_int(get_local_id(2)))) + length_info[(((convert_int(get_group_id(0))) + B) + length_info_elem_offset)]) - length_info[(((B * 2) + (convert_int(get_group_id(0)))) + length_info_elem_offset)]); + } + int seq_offset = condval_3; + int page_no = page_table_values[(((seq_offset >> 4) + cur_page_indptr_begin) + page_values_elem_offset)]; + half4 condval_4; + if ((rotary_mode == 1)) { + float4 freq_1 = (((float4)(((convert_float((((iterator * 8) + k_rope_pos_offset[((convert_int(get_group_id(0))) + k_rope_pos_offset_elem_offset)]) + (convert_int(get_local_id(2)))))) * rope_scale), ((convert_float((((iterator * 8) + k_rope_pos_offset[((convert_int(get_group_id(0))) + k_rope_pos_offset_elem_offset)]) + (convert_int(get_local_id(2)))))) * rope_scale), ((convert_float((((iterator * 8) + k_rope_pos_offset[((convert_int(get_group_id(0))) + k_rope_pos_offset_elem_offset)]) + (convert_int(get_local_id(2)))))) * rope_scale), ((convert_float((((iterator * 8) + k_rope_pos_offset[((convert_int(get_group_id(0))) + k_rope_pos_offset_elem_offset)]) + (convert_int(get_local_id(2)))))) * rope_scale))) / pow(((float4)(rope_theta, rope_theta, rope_theta, rope_theta)), ((convert_float4(((int4)(((((convert_int(get_local_id(0))) & 15) * 8))+(2*0), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*1), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*2), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*3))))) * ((float4)(7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f))))); + int4 v_ = (int4)(((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*0), ((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*1), ((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*2), ((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*3)); + half4 condval_5; + if (((convert_int(get_local_id(0))) < 16)) { + int4 v__1 = (int4)((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 64))+(1*0), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 64))+(1*1), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 64))+(1*2), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 64))+(1*3)); + condval_5 = (((half4)(pages[v__1.s0],pages[v__1.s1],pages[v__1.s2],pages[v__1.s3])) * ((half4)((half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f))); + } else { + int4 v__2 = (int4)((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) - 64))+(1*0), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) - 64))+(1*1), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) - 64))+(1*2), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) - 64))+(1*3)); + condval_5 = ((half4)(pages[v__2.s0],pages[v__2.s1],pages[v__2.s2],pages[v__2.s3])); + } + condval_4 = (convert_half4(((cos(freq_1) * (convert_float4(((half4)(pages[v_.s0],pages[v_.s1],pages[v_.s2],pages[v_.s3]))))) + (sin(freq_1) * (convert_float4(condval_5)))))); + } else { + int4 v__3 = (int4)(((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*0), ((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*1), ((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*2), ((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*3)); + condval_4 = ((half4)(pages[v__3.s0],pages[v__3.s1],pages[v__3.s2],pages[v__3.s3])); + } + K_smem[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = condval_4; + int4 v__4 = (int4)((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 32768))+(1*0), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 32768))+(1*1), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 32768))+(1*2), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 32768))+(1*3)); + V_smem[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = ((half4)(pages[v__4.s0],pages[v__4.s1],pages[v__4.s2],pages[v__4.s3])); + } else { + K_smem[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = ((half4)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)); + V_smem[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = ((half4)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)); + } + barrier(CLK_LOCAL_MEM_FENCE); + m_prev[0] = st_m[0]; + vstore4(((((convert_float4(Q_local[0])) * (convert_float4(K_smem[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))]))) * ((float4)(attn_score_scaling_factor, attn_score_scaling_factor, attn_score_scaling_factor, attn_score_scaling_factor))) * ((float4)(1.275174e-01f, 1.275174e-01f, 1.275174e-01f, 1.275174e-01f))), 0, QK_local + 0); + S_reduce_local[0] = 0.000000e+00f; + S_reduce_local[0] = (S_reduce_local[0] + QK_local[0]); + S_reduce_local[0] = (S_reduce_local[0] + QK_local[1]); + S_reduce_local[0] = (S_reduce_local[0] + QK_local[2]); + S_reduce_local[0] = (S_reduce_local[0] + QK_local[3]); + barrier(CLK_LOCAL_MEM_FENCE); + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = S_reduce_local[0]; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 16) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0)))) + 16)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 8) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0)))) + 8)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 4) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0)))) + 4)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 2) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0)))) + 2)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 1) { + ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0)))) + 1)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + S_local[0] = -5.000000e+04f; + if (((iterator * 8) + (convert_int(get_local_id(2)))) < kv_chunk_len[0]) { + S_local[0] = ((volatile __local float*)red_buf0)[((convert_int(get_local_id(2))) * 32)]; + } + st_m[0] = max((float)st_m[0], (float)S_local[0]); + float o_scale = exp2((m_prev[0] - st_m[0])); + st_d[0] = (st_d[0] * o_scale); + S_local[0] = exp2((S_local[0] - st_m[0])); + st_d[0] = (st_d[0] + S_local[0]); + O_local[0] = (O_local[0] * ((float4)(o_scale, o_scale, o_scale, o_scale))); + V_local[0] = V_smem[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))]; + O_local[0] = (O_local[0] + ((convert_float4(V_local[0])) * ((float4)(S_local[0], S_local[0], S_local[0], S_local[0])))); + } + O_allreduce[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = O_local[0]; + md_allreduce[((convert_int(get_local_id(2))) * 2)] = st_m[0]; + md_allreduce[(((convert_int(get_local_id(2))) * 2) + 1)] = st_d[0]; + barrier(CLK_LOCAL_MEM_FENCE); + st_m[0] = -5.000000e+04f; + st_d[0] = 1.000000e+00f; + O_local[0] = ((float4)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)); + for (int j = 0; j < 8; ++j) { + m_prev[0] = st_m[0]; + d_prev[0] = st_d[0]; + other_m[0] = md_allreduce[(j * 2)]; + other_d[0] = md_allreduce[((j * 2) + 1)]; + other_o[0] = O_allreduce[((j * 32) + (convert_int(get_local_id(0))))]; + st_m[0] = max((float)st_m[0], (float)other_m[0]); + st_d[0] = ((d_prev[0] * exp2((m_prev[0] - st_m[0]))) + (other_d[0] * exp2((other_m[0] - st_m[0])))); + exp_mprev[0] = exp2((m_prev[0] - st_m[0])); + other_m[0] = exp2((other_m[0] - st_m[0])); + O_local[0] = ((O_local[0] * ((float4)(exp_mprev[0], exp_mprev[0], exp_mprev[0], exp_mprev[0]))) + (other_o[0] * ((float4)(other_m[0], other_m[0], other_m[0], other_m[0])))); + } + O_local[0] = (O_local[0] / ((float4)(st_d[0], st_d[0], st_d[0], st_d[0]))); + vstore4((convert_half4(O_local[0])), 0, output + ((((convert_int(get_group_id(0))) * 2048) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); + lse[(((convert_int(get_group_id(0))) * 16) + (convert_int(get_group_id(1))))] = (st_m[0] + log2(st_d[0])); +} + +// Function: gpt_neox_q0f16_copy_single_page_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id) { + if (((convert_int(get_group_id(0))) - (copy_length * 8)) < 0) { + int rmod = ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % (copy_length * 128)); + int rmod_1 = ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % (copy_length * 128)); + int rdiv = ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) / (copy_length * 128)); + int rmod_2 = ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % (copy_length * 128)); + int rmod_3 = ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % (copy_length * 128)); + int rdiv_1 = ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) / (copy_length * 128)); + pages[(((((select((rmod_2 + (copy_length * 128)), rmod_2, (convert_int((((bool)(((bool)((copy_length * 128) >= 0)) && ((bool)(rmod_2 >= 0)))) || ((bool)(((bool)((copy_length * 128) < 0)) && ((bool)(rmod_2 <= 0)))))))) >> 7) * 128) + ((((tgt_page_id * 32) + select((rdiv_1 - 1), rdiv_1, (convert_int((((bool)(((bool)((copy_length * 128) >= 0)) && ((bool)(rmod_3 >= 0)))) || ((bool)(((bool)((copy_length * 128) < 0)) && ((bool)(rmod_3 <= 0))))))))) * page_size) * 128)) + pages_elem_offset) + ((convert_int(get_local_id(0))) & 127))] = pages[(((((select((rmod + (copy_length * 128)), rmod, (convert_int((((bool)(((bool)((copy_length * 128) >= 0)) && ((bool)(rmod >= 0)))) || ((bool)(((bool)((copy_length * 128) < 0)) && ((bool)(rmod <= 0)))))))) >> 7) * 128) + ((((src_page_id * 32) + select((rdiv - 1), rdiv, (convert_int((((bool)(((bool)((copy_length * 128) >= 0)) && ((bool)(rmod_1 >= 0)))) || ((bool)(((bool)((copy_length * 128) < 0)) && ((bool)(rmod_1 <= 0))))))))) * page_size) * 128)) + pages_elem_offset) + ((convert_int(get_local_id(0))) & 127))]; + int rmod_4 = ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % (copy_length * 128)); + int rmod_5 = ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % (copy_length * 128)); + int rdiv_2 = ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) / (copy_length * 128)); + int rmod_6 = ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % (copy_length * 128)); + int rmod_7 = ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % (copy_length * 128)); + int rdiv_3 = ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) / (copy_length * 128)); + pages[(((((select((rmod_6 + (copy_length * 128)), rmod_6, (convert_int((((bool)(((bool)((copy_length * 128) >= 0)) && ((bool)(rmod_6 >= 0)))) || ((bool)(((bool)((copy_length * 128) < 0)) && ((bool)(rmod_6 <= 0)))))))) >> 7) * 128) + (((((tgt_page_id * 32) + select((rdiv_3 - 1), rdiv_3, (convert_int((((bool)(((bool)((copy_length * 128) >= 0)) && ((bool)(rmod_7 >= 0)))) || ((bool)(((bool)((copy_length * 128) < 0)) && ((bool)(rmod_7 <= 0))))))))) + 16) * page_size) * 128)) + pages_elem_offset) + ((convert_int(get_local_id(0))) & 127))] = pages[(((((select((rmod_4 + (copy_length * 128)), rmod_4, (convert_int((((bool)(((bool)((copy_length * 128) >= 0)) && ((bool)(rmod_4 >= 0)))) || ((bool)(((bool)((copy_length * 128) < 0)) && ((bool)(rmod_4 <= 0)))))))) >> 7) * 128) + (((((src_page_id * 32) + select((rdiv_2 - 1), rdiv_2, (convert_int((((bool)(((bool)((copy_length * 128) >= 0)) && ((bool)(rmod_5 >= 0)))) || ((bool)(((bool)((copy_length * 128) < 0)) && ((bool)(rmod_5 <= 0))))))))) + 16) * page_size) * 128)) + pages_elem_offset) + ((convert_int(get_local_id(0))) & 127))]; + } +} + +// Function: gpt_neox_q0f16_fused_reshape8_reshape9_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144) { + T_reshape[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] = add144[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]; +} + +// Function: gpt_neox_q0f16_fused_NT_matmul_add_kernel_2 +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size) { + half8 NT_matmul_intermediate_reindex_pad_local[2]; + for (int var = 0; var < 1; ++var) { + NT_matmul_intermediate_reindex_pad_local[0] = ((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)); + NT_matmul_intermediate_reindex_pad_local[1] = ((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)); + for (int ax3_0 = 0; ax3_0 < 128; ++ax3_0) { + for (int ax3_1 = 0; ax3_1 < 16; ++ax3_1) { + half condval; + if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { + condval = layer_norm147[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; + } else { + condval = (half)0.000000e+00f; + } + int8 v_ = (int8)(((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*0), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*1), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*2), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*3), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*4), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*5), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*6), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*7)); + NT_matmul_intermediate_reindex_pad_local[0] = (NT_matmul_intermediate_reindex_pad_local[0] + (((half8)(condval, condval, condval, condval, condval, condval, condval, condval)) * ((half8)(gpt_neox_layers_0_attention_query_key_value_weight4[v_.s0],gpt_neox_layers_0_attention_query_key_value_weight4[v_.s1],gpt_neox_layers_0_attention_query_key_value_weight4[v_.s2],gpt_neox_layers_0_attention_query_key_value_weight4[v_.s3],gpt_neox_layers_0_attention_query_key_value_weight4[v_.s4],gpt_neox_layers_0_attention_query_key_value_weight4[v_.s5],gpt_neox_layers_0_attention_query_key_value_weight4[v_.s6],gpt_neox_layers_0_attention_query_key_value_weight4[v_.s7])))); + half condval_1; + if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { + condval_1 = layer_norm147[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; + } else { + condval_1 = (half)0.000000e+00f; + } + int8 v__1 = (int8)(((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*0), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*1), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*2), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*3), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*4), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*5), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*6), ((((((convert_int(get_group_id(0))) * 524288) + ((convert_int(get_local_id(0))) * 16384)) + (ax3_0 * 16)) + ax3_1))+(2048*7)); + NT_matmul_intermediate_reindex_pad_local[1] = (NT_matmul_intermediate_reindex_pad_local[1] + (((half8)(condval_1, condval_1, condval_1, condval_1, condval_1, condval_1, condval_1, condval_1)) * ((half8)(gpt_neox_layers_0_attention_query_key_value_weight4[v__1.s0],gpt_neox_layers_0_attention_query_key_value_weight4[v__1.s1],gpt_neox_layers_0_attention_query_key_value_weight4[v__1.s2],gpt_neox_layers_0_attention_query_key_value_weight4[v__1.s3],gpt_neox_layers_0_attention_query_key_value_weight4[v__1.s4],gpt_neox_layers_0_attention_query_key_value_weight4[v__1.s5],gpt_neox_layers_0_attention_query_key_value_weight4[v__1.s6],gpt_neox_layers_0_attention_query_key_value_weight4[v__1.s7])))); + } + } + if ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size) { + vstore8((NT_matmul_intermediate_reindex_pad_local[0] + vload8(0, gpt_neox_layers_0_attention_query_key_value_bias4 + (((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)))), 0, T_add + (((((convert_int(get_group_id(1))) * 49152) + ((convert_int(get_local_id(1))) * 12288)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8))); + } + if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size) { + vstore8((NT_matmul_intermediate_reindex_pad_local[1] + vload8(0, gpt_neox_layers_0_attention_query_key_value_bias4 + (((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)))), 0, T_add + ((((((convert_int(get_group_id(1))) * 49152) + ((convert_int(get_local_id(1))) * 12288)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 6144)); + } + } +} + +// Function: gpt_neox_q0f16_layer_norm2_kernel +#ifdef cl_khr_fp16 +#pragma OPENCL EXTENSION cl_khr_fp16 : enable +#elif defined(cl_amd_fp16) +#pragma OPENCL EXTENSION cl_amd_fp16 : enable +#else +#error "Half precision floating point not supported by OpenCL implementation on your device." +#endif + +__kernel void gpt_neox_q0f16_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv73, int batch_size); +__kernel void gpt_neox_q0f16_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv49, int seq_len); +__kernel void gpt_neox_q0f16_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); +__kernel void gpt_neox_q0f16_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); +__kernel void gpt_neox_q0f16_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_weight3, __global half* restrict take1, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); +__kernel void gpt_neox_q0f16_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); +__kernel void gpt_neox_q0f16_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict astype148, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_weight4, __global half* restrict input_embeds, __global half* restrict reshape291, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight2, __global half* restrict layer_norm49); +__kernel void gpt_neox_q0f16_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict astype49, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight2); +__kernel void gpt_neox_q0f16_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_weight2, __global half* restrict layer_norm97, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); +__kernel void gpt_neox_q0f16_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight3, __global half* restrict layer_norm99, int seq_len); +__kernel void gpt_neox_q0f16_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv25); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); +__kernel void gpt_neox_q0f16_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); +__kernel void gpt_neox_q0f16_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight3, __global half* restrict layer_norm98, int seq_len); +__kernel void gpt_neox_q0f16_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); +__kernel void gpt_neox_q0f16_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); +__kernel void gpt_neox_q0f16_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight2, __global half* restrict layer_norm50); +__kernel void gpt_neox_q0f16_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); +__kernel void gpt_neox_q0f16_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_weight, __global int* restrict input_ids, int seq_len, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict astype98, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight3, int seq_len); +__kernel void gpt_neox_q0f16_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict astype50, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_weight2, __global half* restrict input_embed, __global half* restrict reshape99); +__kernel void gpt_neox_q0f16_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict astype99, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_weight3, __global half* restrict input_embeds, __global half* restrict reshape195, int seq_len); +__kernel void gpt_neox_q0f16_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_weight4, __global half* restrict layer_norm148, int batch_size); +__kernel void gpt_neox_q0f16_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); +__kernel void gpt_neox_q0f16_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict astype147, __global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_weight4, int batch_size); +__kernel void gpt_neox_q0f16_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); +__kernel void gpt_neox_q0f16_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); +__kernel void gpt_neox_q0f16_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); +__kernel void gpt_neox_q0f16_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_weight4, __global half* restrict layer_norm147, int batch_size); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); +__kernel void gpt_neox_q0f16_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed) { + float in_thread_input_embed_red_temp_v0_shared[1]; + float in_thread_input_embed_red_temp_v1_shared[1]; + __local float red_buf0[256]; + __local float red_buf1[256]; + __local float input_embed_red_temp_v0_shared[1]; + __local float input_embed_red_temp_v1_shared[1]; + in_thread_input_embed_red_temp_v0_shared[0] = 0.000000e+00f; + in_thread_input_embed_red_temp_v1_shared[0] = 0.000000e+00f; + float v_input_embed_red_temp_v0 = (in_thread_input_embed_red_temp_v0_shared[0] + (convert_float(input_embed[(convert_int(get_local_id(0)))]))); + float v_input_embed_red_temp_v1 = (in_thread_input_embed_red_temp_v1_shared[0] + ((convert_float(input_embed[(convert_int(get_local_id(0)))])) * (convert_float(input_embed[(convert_int(get_local_id(0)))])))); + in_thread_input_embed_red_temp_v0_shared[0] = v_input_embed_red_temp_v0; + in_thread_input_embed_red_temp_v1_shared[0] = v_input_embed_red_temp_v1; + float v_input_embed_red_temp_v0_1 = (in_thread_input_embed_red_temp_v0_shared[0] + (convert_float(input_embed[((convert_int(get_local_id(0))) + 256)]))); + float v_input_embed_red_temp_v1_1 = (in_thread_input_embed_red_temp_v1_shared[0] + ((convert_float(input_embed[((convert_int(get_local_id(0))) + 256)])) * (convert_float(input_embed[((convert_int(get_local_id(0))) + 256)])))); + in_thread_input_embed_red_temp_v0_shared[0] = v_input_embed_red_temp_v0_1; + in_thread_input_embed_red_temp_v1_shared[0] = v_input_embed_red_temp_v1_1; + float v_input_embed_red_temp_v0_2 = (in_thread_input_embed_red_temp_v0_shared[0] + (convert_float(input_embed[((convert_int(get_local_id(0))) + 512)]))); + float v_input_embed_red_temp_v1_2 = (in_thread_input_embed_red_temp_v1_shared[0] + ((convert_float(input_embed[((convert_int(get_local_id(0))) + 512)])) * (convert_float(input_embed[((convert_int(get_local_id(0))) + 512)])))); + in_thread_input_embed_red_temp_v0_shared[0] = v_input_embed_red_temp_v0_2; + in_thread_input_embed_red_temp_v1_shared[0] = v_input_embed_red_temp_v1_2; + float v_input_embed_red_temp_v0_3 = (in_thread_input_embed_red_temp_v0_shared[0] + (convert_float(input_embed[((convert_int(get_local_id(0))) + 768)]))); + float v_input_embed_red_temp_v1_3 = (in_thread_input_embed_red_temp_v1_shared[0] + ((convert_float(input_embed[((convert_int(get_local_id(0))) + 768)])) * (convert_float(input_embed[((convert_int(get_local_id(0))) + 768)])))); + in_thread_input_embed_red_temp_v0_shared[0] = v_input_embed_red_temp_v0_3; + in_thread_input_embed_red_temp_v1_shared[0] = v_input_embed_red_temp_v1_3; + float v_input_embed_red_temp_v0_4 = (in_thread_input_embed_red_temp_v0_shared[0] + (convert_float(input_embed[((convert_int(get_local_id(0))) + 1024)]))); + float v_input_embed_red_temp_v1_4 = (in_thread_input_embed_red_temp_v1_shared[0] + ((convert_float(input_embed[((convert_int(get_local_id(0))) + 1024)])) * (convert_float(input_embed[((convert_int(get_local_id(0))) + 1024)])))); + in_thread_input_embed_red_temp_v0_shared[0] = v_input_embed_red_temp_v0_4; + in_thread_input_embed_red_temp_v1_shared[0] = v_input_embed_red_temp_v1_4; + float v_input_embed_red_temp_v0_5 = (in_thread_input_embed_red_temp_v0_shared[0] + (convert_float(input_embed[((convert_int(get_local_id(0))) + 1280)]))); + float v_input_embed_red_temp_v1_5 = (in_thread_input_embed_red_temp_v1_shared[0] + ((convert_float(input_embed[((convert_int(get_local_id(0))) + 1280)])) * (convert_float(input_embed[((convert_int(get_local_id(0))) + 1280)])))); + in_thread_input_embed_red_temp_v0_shared[0] = v_input_embed_red_temp_v0_5; + in_thread_input_embed_red_temp_v1_shared[0] = v_input_embed_red_temp_v1_5; + float v_input_embed_red_temp_v0_6 = (in_thread_input_embed_red_temp_v0_shared[0] + (convert_float(input_embed[((convert_int(get_local_id(0))) + 1536)]))); + float v_input_embed_red_temp_v1_6 = (in_thread_input_embed_red_temp_v1_shared[0] + ((convert_float(input_embed[((convert_int(get_local_id(0))) + 1536)])) * (convert_float(input_embed[((convert_int(get_local_id(0))) + 1536)])))); + in_thread_input_embed_red_temp_v0_shared[0] = v_input_embed_red_temp_v0_6; + in_thread_input_embed_red_temp_v1_shared[0] = v_input_embed_red_temp_v1_6; + float v_input_embed_red_temp_v0_7 = (in_thread_input_embed_red_temp_v0_shared[0] + (convert_float(input_embed[((convert_int(get_local_id(0))) + 1792)]))); + float v_input_embed_red_temp_v1_7 = (in_thread_input_embed_red_temp_v1_shared[0] + ((convert_float(input_embed[((convert_int(get_local_id(0))) + 1792)])) * (convert_float(input_embed[((convert_int(get_local_id(0))) + 1792)])))); + in_thread_input_embed_red_temp_v0_shared[0] = v_input_embed_red_temp_v0_7; + in_thread_input_embed_red_temp_v1_shared[0] = v_input_embed_red_temp_v1_7; + barrier(CLK_LOCAL_MEM_FENCE); + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = in_thread_input_embed_red_temp_v0_shared[0]; + ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = in_thread_input_embed_red_temp_v1_shared[0]; + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 128) { + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 128)]); + ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 128)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 64) { + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 64)]); + ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 64)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 32) { + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 32)]); + ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 32)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 16) { + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 16)]); + ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 16)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 8) { + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 8)]); + ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 8)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 4) { + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 4)]); + ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 4)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 2) { + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 2)]); + ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 2)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) < 1) { + ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 1)]); + ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 1)]); + } + barrier(CLK_LOCAL_MEM_FENCE); + if ((convert_int(get_local_id(0))) == 0) { + input_embed_red_temp_v0_shared[0] = ((volatile __local float*)red_buf0)[0]; + input_embed_red_temp_v1_shared[0] = ((volatile __local float*)red_buf1)[0]; + } + barrier(CLK_LOCAL_MEM_FENCE); + T_layer_norm[(convert_int(get_local_id(0)))] = (((convert_half((((convert_float(input_embed[(convert_int(get_local_id(0)))])) - (input_embed_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embed_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embed_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embed_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight2[(convert_int(get_local_id(0)))]) + gpt_neox_layers_0_input_layernorm_bias2[(convert_int(get_local_id(0)))]); + T_layer_norm[((convert_int(get_local_id(0))) + 256)] = (((convert_half((((convert_float(input_embed[((convert_int(get_local_id(0))) + 256)])) - (input_embed_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embed_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embed_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embed_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight2[((convert_int(get_local_id(0))) + 256)]) + gpt_neox_layers_0_input_layernorm_bias2[((convert_int(get_local_id(0))) + 256)]); + T_layer_norm[((convert_int(get_local_id(0))) + 512)] = (((convert_half((((convert_float(input_embed[((convert_int(get_local_id(0))) + 512)])) - (input_embed_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embed_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embed_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embed_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight2[((convert_int(get_local_id(0))) + 512)]) + gpt_neox_layers_0_input_layernorm_bias2[((convert_int(get_local_id(0))) + 512)]); + T_layer_norm[((convert_int(get_local_id(0))) + 768)] = (((convert_half((((convert_float(input_embed[((convert_int(get_local_id(0))) + 768)])) - (input_embed_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embed_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embed_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embed_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight2[((convert_int(get_local_id(0))) + 768)]) + gpt_neox_layers_0_input_layernorm_bias2[((convert_int(get_local_id(0))) + 768)]); + T_layer_norm[((convert_int(get_local_id(0))) + 1024)] = (((convert_half((((convert_float(input_embed[((convert_int(get_local_id(0))) + 1024)])) - (input_embed_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embed_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embed_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embed_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight2[((convert_int(get_local_id(0))) + 1024)]) + gpt_neox_layers_0_input_layernorm_bias2[((convert_int(get_local_id(0))) + 1024)]); + T_layer_norm[((convert_int(get_local_id(0))) + 1280)] = (((convert_half((((convert_float(input_embed[((convert_int(get_local_id(0))) + 1280)])) - (input_embed_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embed_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embed_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embed_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight2[((convert_int(get_local_id(0))) + 1280)]) + gpt_neox_layers_0_input_layernorm_bias2[((convert_int(get_local_id(0))) + 1280)]); + T_layer_norm[((convert_int(get_local_id(0))) + 1536)] = (((convert_half((((convert_float(input_embed[((convert_int(get_local_id(0))) + 1536)])) - (input_embed_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embed_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embed_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embed_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight2[((convert_int(get_local_id(0))) + 1536)]) + gpt_neox_layers_0_input_layernorm_bias2[((convert_int(get_local_id(0))) + 1536)]); + T_layer_norm[((convert_int(get_local_id(0))) + 1792)] = (((convert_half((((convert_float(input_embed[((convert_int(get_local_id(0))) + 1792)])) - (input_embed_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embed_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embed_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embed_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight2[((convert_int(get_local_id(0))) + 1792)]) + gpt_neox_layers_0_input_layernorm_bias2[((convert_int(get_local_id(0))) + 1792)]); +} +