// Function: gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset) { if (position_map[(((convert_int(get_group_id(0))) >> 3) + position_map_elem_offset)] != -1) { int position = position_map[(((convert_int(get_group_id(0))) >> 3) + position_map_elem_offset)]; pages[(((((((position >> 4) * 65536) + (((convert_int(get_group_id(0))) & 7) * 4096)) + (((convert_int(get_local_id(0))) >> 7) * 2048)) + ((position & 15) * 128)) + pages_elem_offset) + ((convert_int(get_local_id(0))) & 127))] = k_data[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]; int position_1 = position_map[(((convert_int(get_group_id(0))) >> 3) + position_map_elem_offset)]; pages[((((((((position_1 >> 4) * 65536) + (((convert_int(get_group_id(0))) & 7) * 4096)) + (((convert_int(get_local_id(0))) >> 7) * 2048)) + ((position_1 & 15) * 128)) + pages_elem_offset) + ((convert_int(get_local_id(0))) & 127)) + 32768)] = v_data[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]; } } // Function: gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2 #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size) { float NT_matmul_intermediate_reindex_pad_local[16]; for (int var = 0; var < 1; ++var) { vstore8(((float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)), 0, NT_matmul_intermediate_reindex_pad_local + 0); vstore8(((float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)), 0, NT_matmul_intermediate_reindex_pad_local + 8); for (int ax3_0 = 0; ax3_0 < 128; ++ax3_0) { for (int ax3_1 = 0; ax3_1 < 16; ++ax3_1) { uint cse_var_9 = (convert_uint(((ax3_1 & 7) * 4))); half condval; if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { condval = layer_norm148[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; } else { condval = (half)0.000000e+00f; } int8 v_ = (int8)(((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*0), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*1), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*2), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*3), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*4), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*5), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*6), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*7)); int8 v__1 = (int8)((((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*0), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*1), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*2), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*3), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*4), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*5), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*6), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*7)); vstore8((((convert_float8(((half8)(condval, condval, condval, condval, condval, condval, condval, condval)))) * (convert_float8((((convert_half8(((((uint8)(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[v_.s0],gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[v_.s1],gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[v_.s2],gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[v_.s3],gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[v_.s4],gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[v_.s5],gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[v_.s6],gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[v_.s7])) >> ((uint8)(cse_var_9, cse_var_9, cse_var_9, cse_var_9, cse_var_9, cse_var_9, cse_var_9, cse_var_9))) & ((uint8)((uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15))))) - ((half8)((half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f))) * ((half8)(gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[v__1.s0],gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[v__1.s1],gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[v__1.s2],gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[v__1.s3],gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[v__1.s4],gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[v__1.s5],gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[v__1.s6],gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[v__1.s7])))))) + vload8(0, NT_matmul_intermediate_reindex_pad_local + 0)), 0, NT_matmul_intermediate_reindex_pad_local + 0); half condval_1; if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { condval_1 = layer_norm148[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; } else { condval_1 = (half)0.000000e+00f; } int8 v__2 = (int8)(((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*0), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*1), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*2), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*3), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*4), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*5), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*6), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*7)); int8 v__3 = (int8)((((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*0), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*1), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*2), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*3), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*4), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*5), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*6), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*7)); vstore8((((convert_float8(((half8)(condval_1, condval_1, condval_1, condval_1, condval_1, condval_1, condval_1, condval_1)))) * (convert_float8((((convert_half8(((((uint8)(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[v__2.s0],gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[v__2.s1],gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[v__2.s2],gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[v__2.s3],gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[v__2.s4],gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[v__2.s5],gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[v__2.s6],gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[v__2.s7])) >> ((uint8)(cse_var_9, cse_var_9, cse_var_9, cse_var_9, cse_var_9, cse_var_9, cse_var_9, cse_var_9))) & ((uint8)((uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15))))) - ((half8)((half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f))) * ((half8)(gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[v__3.s0],gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[v__3.s1],gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[v__3.s2],gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[v__3.s3],gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[v__3.s4],gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[v__3.s5],gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[v__3.s6],gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[v__3.s7])))))) + vload8(0, NT_matmul_intermediate_reindex_pad_local + 8)), 0, NT_matmul_intermediate_reindex_pad_local + 8); } } if ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size) { compute[(((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8))] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[0] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[(((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8))]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[0] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[(((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8))]) * 7.071068e-01f)) * 5.000000e-01f))))); compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 1)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[1] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 1)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[1] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 1)]) * 7.071068e-01f)) * 5.000000e-01f))))); compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 2)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[2] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 2)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[2] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 2)]) * 7.071068e-01f)) * 5.000000e-01f))))); compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 3)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[3] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 3)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[3] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 3)]) * 7.071068e-01f)) * 5.000000e-01f))))); compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 4)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[4] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 4)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[4] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 4)]) * 7.071068e-01f)) * 5.000000e-01f))))); compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 5)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[5] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 5)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[5] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 5)]) * 7.071068e-01f)) * 5.000000e-01f))))); compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 6)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[6] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 6)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[6] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 6)]) * 7.071068e-01f)) * 5.000000e-01f))))); compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 7)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[7] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 7)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[7] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 7)]) * 7.071068e-01f)) * 5.000000e-01f))))); } if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size) { compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 8192)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[8] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[(((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8))]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[8] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[(((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8))]) * 7.071068e-01f)) * 5.000000e-01f))))); compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 8193)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[9] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 1)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[9] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 1)]) * 7.071068e-01f)) * 5.000000e-01f))))); compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 8194)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[10] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 2)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[10] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 2)]) * 7.071068e-01f)) * 5.000000e-01f))))); compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 8195)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[11] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 3)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[11] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 3)]) * 7.071068e-01f)) * 5.000000e-01f))))); compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 8196)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[12] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 4)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[12] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 4)]) * 7.071068e-01f)) * 5.000000e-01f))))); compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 8197)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[13] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 5)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[13] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 5)]) * 7.071068e-01f)) * 5.000000e-01f))))); compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 8198)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[14] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 6)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[14] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 6)]) * 7.071068e-01f)) * 5.000000e-01f))))); compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 8199)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[15] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 7)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[15] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 7)]) * 7.071068e-01f)) * 5.000000e-01f))))); } } } // Function: gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1 #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len) { half NT_matmul_intermediate_pad_rf_local[64]; half dequantize_local[16]; half NT_matmul_intermediate_pad_rf_local_1[8]; half NT_matmul_intermediate_pad_local[8]; __local half red_buf0[64]; for (int ax0_1_init = 0; ax0_1_init < 4; ++ax0_1_init) { for (int ax1_fused_2_init = 0; ax1_fused_2_init < 2; ++ax1_fused_2_init) { int8 v_ = (int8)((((ax0_1_init * 2) + ax1_fused_2_init))+(8*0), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*1), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*2), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*3), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*4), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*5), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*6), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*7)); half8 v__1 = (half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f); NT_matmul_intermediate_pad_rf_local[v_.s0] = v__1.s0; NT_matmul_intermediate_pad_rf_local[v_.s1] = v__1.s1; NT_matmul_intermediate_pad_rf_local[v_.s2] = v__1.s2; NT_matmul_intermediate_pad_rf_local[v_.s3] = v__1.s3; NT_matmul_intermediate_pad_rf_local[v_.s4] = v__1.s4; NT_matmul_intermediate_pad_rf_local[v_.s5] = v__1.s5; NT_matmul_intermediate_pad_rf_local[v_.s6] = v__1.s6; NT_matmul_intermediate_pad_rf_local[v_.s7] = v__1.s7; } } for (int ax2_fused_0 = 0; ax2_fused_0 < 8; ++ax2_fused_0) { for (int ax0_0 = 0; ax0_0 < 2; ++ax0_0) { dequantize_local[(ax0_0 * 8)] = (((convert_half(((gpt_neox_layers_0_attention_dense_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)0) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_dense_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 1)] = (((convert_half(((gpt_neox_layers_0_attention_dense_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)4) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_dense_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 2)] = (((convert_half(((gpt_neox_layers_0_attention_dense_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)8) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_dense_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 3)] = (((convert_half(((gpt_neox_layers_0_attention_dense_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)12) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_dense_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 4)] = (((convert_half(((gpt_neox_layers_0_attention_dense_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)16) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_dense_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 5)] = (((convert_half(((gpt_neox_layers_0_attention_dense_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)20) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_dense_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 6)] = (((convert_half(((gpt_neox_layers_0_attention_dense_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)24) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_dense_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 7)] = (((convert_half(((gpt_neox_layers_0_attention_dense_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)28) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_dense_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); } if (((convert_int(get_group_id(1))) * 4) < seq_len) { int8 v__2 = (int8)((0)+(8*0), (0)+(8*1), (0)+(8*2), (0)+(8*3), (0)+(8*4), (0)+(8*5), (0)+(8*6), (0)+(8*7)); half8 v__3 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__2.s0],NT_matmul_intermediate_pad_rf_local[v__2.s1],NT_matmul_intermediate_pad_rf_local[v__2.s2],NT_matmul_intermediate_pad_rf_local[v__2.s3],NT_matmul_intermediate_pad_rf_local[v__2.s4],NT_matmul_intermediate_pad_rf_local[v__2.s5],NT_matmul_intermediate_pad_rf_local[v__2.s6],NT_matmul_intermediate_pad_rf_local[v__2.s7])) + (vload8(0, reshape195 + ((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, dequantize_local + 0)); NT_matmul_intermediate_pad_rf_local[v__2.s0] = v__3.s0; NT_matmul_intermediate_pad_rf_local[v__2.s1] = v__3.s1; NT_matmul_intermediate_pad_rf_local[v__2.s2] = v__3.s2; NT_matmul_intermediate_pad_rf_local[v__2.s3] = v__3.s3; NT_matmul_intermediate_pad_rf_local[v__2.s4] = v__3.s4; NT_matmul_intermediate_pad_rf_local[v__2.s5] = v__3.s5; NT_matmul_intermediate_pad_rf_local[v__2.s6] = v__3.s6; NT_matmul_intermediate_pad_rf_local[v__2.s7] = v__3.s7; int8 v__4 = (int8)((1)+(8*0), (1)+(8*1), (1)+(8*2), (1)+(8*3), (1)+(8*4), (1)+(8*5), (1)+(8*6), (1)+(8*7)); half8 v__5 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__4.s0],NT_matmul_intermediate_pad_rf_local[v__4.s1],NT_matmul_intermediate_pad_rf_local[v__4.s2],NT_matmul_intermediate_pad_rf_local[v__4.s3],NT_matmul_intermediate_pad_rf_local[v__4.s4],NT_matmul_intermediate_pad_rf_local[v__4.s5],NT_matmul_intermediate_pad_rf_local[v__4.s6],NT_matmul_intermediate_pad_rf_local[v__4.s7])) + (vload8(0, reshape195 + ((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, dequantize_local + 8)); NT_matmul_intermediate_pad_rf_local[v__4.s0] = v__5.s0; NT_matmul_intermediate_pad_rf_local[v__4.s1] = v__5.s1; NT_matmul_intermediate_pad_rf_local[v__4.s2] = v__5.s2; NT_matmul_intermediate_pad_rf_local[v__4.s3] = v__5.s3; NT_matmul_intermediate_pad_rf_local[v__4.s4] = v__5.s4; NT_matmul_intermediate_pad_rf_local[v__4.s5] = v__5.s5; NT_matmul_intermediate_pad_rf_local[v__4.s6] = v__5.s6; NT_matmul_intermediate_pad_rf_local[v__4.s7] = v__5.s7; } if ((((convert_int(get_group_id(1))) * 4) + 1) < seq_len) { int8 v__6 = (int8)((2)+(8*0), (2)+(8*1), (2)+(8*2), (2)+(8*3), (2)+(8*4), (2)+(8*5), (2)+(8*6), (2)+(8*7)); half8 v__7 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__6.s0],NT_matmul_intermediate_pad_rf_local[v__6.s1],NT_matmul_intermediate_pad_rf_local[v__6.s2],NT_matmul_intermediate_pad_rf_local[v__6.s3],NT_matmul_intermediate_pad_rf_local[v__6.s4],NT_matmul_intermediate_pad_rf_local[v__6.s5],NT_matmul_intermediate_pad_rf_local[v__6.s6],NT_matmul_intermediate_pad_rf_local[v__6.s7])) + (vload8(0, reshape195 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)) * vload8(0, dequantize_local + 0)); NT_matmul_intermediate_pad_rf_local[v__6.s0] = v__7.s0; NT_matmul_intermediate_pad_rf_local[v__6.s1] = v__7.s1; NT_matmul_intermediate_pad_rf_local[v__6.s2] = v__7.s2; NT_matmul_intermediate_pad_rf_local[v__6.s3] = v__7.s3; NT_matmul_intermediate_pad_rf_local[v__6.s4] = v__7.s4; NT_matmul_intermediate_pad_rf_local[v__6.s5] = v__7.s5; NT_matmul_intermediate_pad_rf_local[v__6.s6] = v__7.s6; NT_matmul_intermediate_pad_rf_local[v__6.s7] = v__7.s7; int8 v__8 = (int8)((3)+(8*0), (3)+(8*1), (3)+(8*2), (3)+(8*3), (3)+(8*4), (3)+(8*5), (3)+(8*6), (3)+(8*7)); half8 v__9 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__8.s0],NT_matmul_intermediate_pad_rf_local[v__8.s1],NT_matmul_intermediate_pad_rf_local[v__8.s2],NT_matmul_intermediate_pad_rf_local[v__8.s3],NT_matmul_intermediate_pad_rf_local[v__8.s4],NT_matmul_intermediate_pad_rf_local[v__8.s5],NT_matmul_intermediate_pad_rf_local[v__8.s6],NT_matmul_intermediate_pad_rf_local[v__8.s7])) + (vload8(0, reshape195 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)) * vload8(0, dequantize_local + 8)); NT_matmul_intermediate_pad_rf_local[v__8.s0] = v__9.s0; NT_matmul_intermediate_pad_rf_local[v__8.s1] = v__9.s1; NT_matmul_intermediate_pad_rf_local[v__8.s2] = v__9.s2; NT_matmul_intermediate_pad_rf_local[v__8.s3] = v__9.s3; NT_matmul_intermediate_pad_rf_local[v__8.s4] = v__9.s4; NT_matmul_intermediate_pad_rf_local[v__8.s5] = v__9.s5; NT_matmul_intermediate_pad_rf_local[v__8.s6] = v__9.s6; NT_matmul_intermediate_pad_rf_local[v__8.s7] = v__9.s7; } if ((((convert_int(get_group_id(1))) * 4) + 2) < seq_len) { int8 v__10 = (int8)((4)+(8*0), (4)+(8*1), (4)+(8*2), (4)+(8*3), (4)+(8*4), (4)+(8*5), (4)+(8*6), (4)+(8*7)); half8 v__11 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__10.s0],NT_matmul_intermediate_pad_rf_local[v__10.s1],NT_matmul_intermediate_pad_rf_local[v__10.s2],NT_matmul_intermediate_pad_rf_local[v__10.s3],NT_matmul_intermediate_pad_rf_local[v__10.s4],NT_matmul_intermediate_pad_rf_local[v__10.s5],NT_matmul_intermediate_pad_rf_local[v__10.s6],NT_matmul_intermediate_pad_rf_local[v__10.s7])) + (vload8(0, reshape195 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 4096)) * vload8(0, dequantize_local + 0)); NT_matmul_intermediate_pad_rf_local[v__10.s0] = v__11.s0; NT_matmul_intermediate_pad_rf_local[v__10.s1] = v__11.s1; NT_matmul_intermediate_pad_rf_local[v__10.s2] = v__11.s2; NT_matmul_intermediate_pad_rf_local[v__10.s3] = v__11.s3; NT_matmul_intermediate_pad_rf_local[v__10.s4] = v__11.s4; NT_matmul_intermediate_pad_rf_local[v__10.s5] = v__11.s5; NT_matmul_intermediate_pad_rf_local[v__10.s6] = v__11.s6; NT_matmul_intermediate_pad_rf_local[v__10.s7] = v__11.s7; int8 v__12 = (int8)((5)+(8*0), (5)+(8*1), (5)+(8*2), (5)+(8*3), (5)+(8*4), (5)+(8*5), (5)+(8*6), (5)+(8*7)); half8 v__13 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__12.s0],NT_matmul_intermediate_pad_rf_local[v__12.s1],NT_matmul_intermediate_pad_rf_local[v__12.s2],NT_matmul_intermediate_pad_rf_local[v__12.s3],NT_matmul_intermediate_pad_rf_local[v__12.s4],NT_matmul_intermediate_pad_rf_local[v__12.s5],NT_matmul_intermediate_pad_rf_local[v__12.s6],NT_matmul_intermediate_pad_rf_local[v__12.s7])) + (vload8(0, reshape195 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 4096)) * vload8(0, dequantize_local + 8)); NT_matmul_intermediate_pad_rf_local[v__12.s0] = v__13.s0; NT_matmul_intermediate_pad_rf_local[v__12.s1] = v__13.s1; NT_matmul_intermediate_pad_rf_local[v__12.s2] = v__13.s2; NT_matmul_intermediate_pad_rf_local[v__12.s3] = v__13.s3; NT_matmul_intermediate_pad_rf_local[v__12.s4] = v__13.s4; NT_matmul_intermediate_pad_rf_local[v__12.s5] = v__13.s5; NT_matmul_intermediate_pad_rf_local[v__12.s6] = v__13.s6; NT_matmul_intermediate_pad_rf_local[v__12.s7] = v__13.s7; } if ((((convert_int(get_group_id(1))) * 4) + 3) < seq_len) { int8 v__14 = (int8)((6)+(8*0), (6)+(8*1), (6)+(8*2), (6)+(8*3), (6)+(8*4), (6)+(8*5), (6)+(8*6), (6)+(8*7)); half8 v__15 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__14.s0],NT_matmul_intermediate_pad_rf_local[v__14.s1],NT_matmul_intermediate_pad_rf_local[v__14.s2],NT_matmul_intermediate_pad_rf_local[v__14.s3],NT_matmul_intermediate_pad_rf_local[v__14.s4],NT_matmul_intermediate_pad_rf_local[v__14.s5],NT_matmul_intermediate_pad_rf_local[v__14.s6],NT_matmul_intermediate_pad_rf_local[v__14.s7])) + (vload8(0, reshape195 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 6144)) * vload8(0, dequantize_local + 0)); NT_matmul_intermediate_pad_rf_local[v__14.s0] = v__15.s0; NT_matmul_intermediate_pad_rf_local[v__14.s1] = v__15.s1; NT_matmul_intermediate_pad_rf_local[v__14.s2] = v__15.s2; NT_matmul_intermediate_pad_rf_local[v__14.s3] = v__15.s3; NT_matmul_intermediate_pad_rf_local[v__14.s4] = v__15.s4; NT_matmul_intermediate_pad_rf_local[v__14.s5] = v__15.s5; NT_matmul_intermediate_pad_rf_local[v__14.s6] = v__15.s6; NT_matmul_intermediate_pad_rf_local[v__14.s7] = v__15.s7; int8 v__16 = (int8)((7)+(8*0), (7)+(8*1), (7)+(8*2), (7)+(8*3), (7)+(8*4), (7)+(8*5), (7)+(8*6), (7)+(8*7)); half8 v__17 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__16.s0],NT_matmul_intermediate_pad_rf_local[v__16.s1],NT_matmul_intermediate_pad_rf_local[v__16.s2],NT_matmul_intermediate_pad_rf_local[v__16.s3],NT_matmul_intermediate_pad_rf_local[v__16.s4],NT_matmul_intermediate_pad_rf_local[v__16.s5],NT_matmul_intermediate_pad_rf_local[v__16.s6],NT_matmul_intermediate_pad_rf_local[v__16.s7])) + (vload8(0, reshape195 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 6144)) * vload8(0, dequantize_local + 8)); NT_matmul_intermediate_pad_rf_local[v__16.s0] = v__17.s0; NT_matmul_intermediate_pad_rf_local[v__16.s1] = v__17.s1; NT_matmul_intermediate_pad_rf_local[v__16.s2] = v__17.s2; NT_matmul_intermediate_pad_rf_local[v__16.s3] = v__17.s3; NT_matmul_intermediate_pad_rf_local[v__16.s4] = v__17.s4; NT_matmul_intermediate_pad_rf_local[v__16.s5] = v__17.s5; NT_matmul_intermediate_pad_rf_local[v__16.s6] = v__17.s6; NT_matmul_intermediate_pad_rf_local[v__16.s7] = v__17.s7; } } for (int ax3_fused_2_0 = 0; ax3_fused_2_0 < 1; ++ax3_fused_2_0) { for (int ax2 = 0; ax2 < 4; ++ax2) { vstore2(((half2)((half)0.000000e+00f, (half)0.000000e+00f)), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + (ax2 * 2))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 8))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 16))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 24))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 32))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 40))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 48))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 56))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); } } for (int ax2_fused_2 = 0; ax2_fused_2 < 2; ++ax2_fused_2) { for (int ax1 = 0; ax1 < 4; ++ax1) { barrier(CLK_LOCAL_MEM_FENCE); ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_intermediate_pad_rf_local_1[((ax1 * 2) + ax2_fused_2)]; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 16) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 8) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 4) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 2) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); } barrier(CLK_LOCAL_MEM_FENCE); NT_matmul_intermediate_pad_local[((ax1 * 2) + ax2_fused_2)] = ((volatile __local half*)red_buf0)[((convert_int(get_local_id(0))) * 32)]; } } for (int ax0 = 0; ax0 < 4; ++ax0) { for (int ax1_fused_2 = 0; ax1_fused_2 < 2; ++ax1_fused_2) { if (((bool)(((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)(((bool)(((convert_int(get_group_id(1))) - ((seq_len + 3) >> 2)) < 0)) || ((bool)((convert_int(get_group_id(1))) == 0)))))) && ((bool)((((convert_int(get_group_id(1))) * 4) + ax0) < seq_len))) { T_add[((((((convert_int(get_group_id(1))) * 8192) + (ax0 * 2048)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] = ((lv74[((((((convert_int(get_group_id(1))) * 8192) + (ax0 * 2048)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] + (NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_attention_dense_bias3[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)])) + input_embeds[((((((convert_int(get_group_id(1))) * 8192) + (ax0 * 2048)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]); } } } } // Function: gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2 #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len) { float NT_matmul_intermediate_reindex_pad_local[16]; for (int var = 0; var < 1; ++var) { vstore8(((float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)), 0, NT_matmul_intermediate_reindex_pad_local + 0); vstore8(((float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)), 0, NT_matmul_intermediate_reindex_pad_local + 8); for (int ax3_0 = 0; ax3_0 < 128; ++ax3_0) { for (int ax3_1 = 0; ax3_1 < 16; ++ax3_1) { uint cse_var_9 = (convert_uint(((ax3_1 & 7) * 4))); half condval; if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < seq_len)) { condval = layer_norm99[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; } else { condval = (half)0.000000e+00f; } int8 v_ = (int8)(((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*0), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*1), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*2), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*3), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*4), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*5), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*6), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*7)); int8 v__1 = (int8)((((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*0), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*1), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*2), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*3), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*4), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*5), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*6), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*7)); vstore8((((convert_float8(((half8)(condval, condval, condval, condval, condval, condval, condval, condval)))) * (convert_float8((((convert_half8(((((uint8)(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[v_.s0],gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[v_.s1],gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[v_.s2],gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[v_.s3],gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[v_.s4],gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[v_.s5],gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[v_.s6],gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[v_.s7])) >> ((uint8)(cse_var_9, cse_var_9, cse_var_9, cse_var_9, cse_var_9, cse_var_9, cse_var_9, cse_var_9))) & ((uint8)((uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15))))) - ((half8)((half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f))) * ((half8)(gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[v__1.s0],gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[v__1.s1],gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[v__1.s2],gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[v__1.s3],gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[v__1.s4],gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[v__1.s5],gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[v__1.s6],gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[v__1.s7])))))) + vload8(0, NT_matmul_intermediate_reindex_pad_local + 0)), 0, NT_matmul_intermediate_reindex_pad_local + 0); half condval_1; if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < seq_len)) { condval_1 = layer_norm99[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; } else { condval_1 = (half)0.000000e+00f; } int8 v__2 = (int8)(((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*0), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*1), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*2), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*3), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*4), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*5), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*6), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*7)); int8 v__3 = (int8)((((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*0), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*1), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*2), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*3), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*4), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*5), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*6), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*7)); vstore8((((convert_float8(((half8)(condval_1, condval_1, condval_1, condval_1, condval_1, condval_1, condval_1, condval_1)))) * (convert_float8((((convert_half8(((((uint8)(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[v__2.s0],gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[v__2.s1],gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[v__2.s2],gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[v__2.s3],gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[v__2.s4],gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[v__2.s5],gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[v__2.s6],gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[v__2.s7])) >> ((uint8)(cse_var_9, cse_var_9, cse_var_9, cse_var_9, cse_var_9, cse_var_9, cse_var_9, cse_var_9))) & ((uint8)((uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15))))) - ((half8)((half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f))) * ((half8)(gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[v__3.s0],gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[v__3.s1],gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[v__3.s2],gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[v__3.s3],gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[v__3.s4],gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[v__3.s5],gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[v__3.s6],gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[v__3.s7])))))) + vload8(0, NT_matmul_intermediate_reindex_pad_local + 8)), 0, NT_matmul_intermediate_reindex_pad_local + 8); } } if ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < seq_len) { compute[(((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8))] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[0] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[(((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8))]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[0] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[(((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8))]) * 7.071068e-01f)) * 5.000000e-01f))))); compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 1)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[1] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 1)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[1] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 1)]) * 7.071068e-01f)) * 5.000000e-01f))))); compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 2)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[2] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 2)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[2] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 2)]) * 7.071068e-01f)) * 5.000000e-01f))))); compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 3)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[3] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 3)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[3] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 3)]) * 7.071068e-01f)) * 5.000000e-01f))))); compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 4)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[4] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 4)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[4] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 4)]) * 7.071068e-01f)) * 5.000000e-01f))))); compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 5)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[5] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 5)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[5] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 5)]) * 7.071068e-01f)) * 5.000000e-01f))))); compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 6)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[6] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 6)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[6] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 6)]) * 7.071068e-01f)) * 5.000000e-01f))))); compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 7)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[7] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 7)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[7] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 7)]) * 7.071068e-01f)) * 5.000000e-01f))))); } if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < seq_len) { compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 8192)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[8] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[(((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8))]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[8] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[(((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8))]) * 7.071068e-01f)) * 5.000000e-01f))))); compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 8193)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[9] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 1)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[9] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 1)]) * 7.071068e-01f)) * 5.000000e-01f))))); compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 8194)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[10] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 2)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[10] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 2)]) * 7.071068e-01f)) * 5.000000e-01f))))); compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 8195)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[11] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 3)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[11] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 3)]) * 7.071068e-01f)) * 5.000000e-01f))))); compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 8196)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[12] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 4)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[12] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 4)]) * 7.071068e-01f)) * 5.000000e-01f))))); compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 8197)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[13] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 5)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[13] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 5)]) * 7.071068e-01f)) * 5.000000e-01f))))); compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 8198)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[14] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 6)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[14] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 6)]) * 7.071068e-01f)) * 5.000000e-01f))))); compute[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 8199)] = (convert_half(((NT_matmul_intermediate_reindex_pad_local[15] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 7)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_reindex_pad_local[15] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 7)]) * 7.071068e-01f)) * 5.000000e-01f))))); } } } // Function: gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1 #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size) { float NT_matmul_intermediate_pad_rf_local[64]; half dequantize_local[16]; float NT_matmul_intermediate_pad_rf_local_1[8]; float NT_matmul_intermediate_pad_local[8]; __local float red_buf0[64]; for (int ax0_1_init = 0; ax0_1_init < 4; ++ax0_1_init) { for (int ax1_fused_2_init = 0; ax1_fused_2_init < 2; ++ax1_fused_2_init) { int8 v_ = (int8)((((ax0_1_init * 2) + ax1_fused_2_init))+(8*0), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*1), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*2), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*3), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*4), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*5), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*6), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*7)); float8 v__1 = (float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f); NT_matmul_intermediate_pad_rf_local[v_.s0] = v__1.s0; NT_matmul_intermediate_pad_rf_local[v_.s1] = v__1.s1; NT_matmul_intermediate_pad_rf_local[v_.s2] = v__1.s2; NT_matmul_intermediate_pad_rf_local[v_.s3] = v__1.s3; NT_matmul_intermediate_pad_rf_local[v_.s4] = v__1.s4; NT_matmul_intermediate_pad_rf_local[v_.s5] = v__1.s5; NT_matmul_intermediate_pad_rf_local[v_.s6] = v__1.s6; NT_matmul_intermediate_pad_rf_local[v_.s7] = v__1.s7; } } for (int ax2_fused_0 = 0; ax2_fused_0 < 32; ++ax2_fused_0) { for (int ax0_0 = 0; ax0_0 < 2; ++ax0_0) { dequantize_local[(ax0_0 * 8)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax0_0 * 1024)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)0) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 1)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax0_0 * 1024)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)4) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 2)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax0_0 * 1024)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)8) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 3)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax0_0 * 1024)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)12) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 4)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax0_0 * 1024)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)16) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 5)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax0_0 * 1024)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)20) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 6)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax0_0 * 1024)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)24) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 7)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax0_0 * 1024)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)28) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); } if (((convert_int(get_group_id(1))) * 4) < batch_size) { int8 v__2 = (int8)((0)+(8*0), (0)+(8*1), (0)+(8*2), (0)+(8*3), (0)+(8*4), (0)+(8*5), (0)+(8*6), (0)+(8*7)); float8 v__3 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__2.s0],NT_matmul_intermediate_pad_rf_local[v__2.s1],NT_matmul_intermediate_pad_rf_local[v__2.s2],NT_matmul_intermediate_pad_rf_local[v__2.s3],NT_matmul_intermediate_pad_rf_local[v__2.s4],NT_matmul_intermediate_pad_rf_local[v__2.s5],NT_matmul_intermediate_pad_rf_local[v__2.s6],NT_matmul_intermediate_pad_rf_local[v__2.s7])) + ((convert_float8(vload8(0, lv1 + ((((convert_int(get_group_id(1))) * 32768) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, dequantize_local + 0)))); NT_matmul_intermediate_pad_rf_local[v__2.s0] = v__3.s0; NT_matmul_intermediate_pad_rf_local[v__2.s1] = v__3.s1; NT_matmul_intermediate_pad_rf_local[v__2.s2] = v__3.s2; NT_matmul_intermediate_pad_rf_local[v__2.s3] = v__3.s3; NT_matmul_intermediate_pad_rf_local[v__2.s4] = v__3.s4; NT_matmul_intermediate_pad_rf_local[v__2.s5] = v__3.s5; NT_matmul_intermediate_pad_rf_local[v__2.s6] = v__3.s6; NT_matmul_intermediate_pad_rf_local[v__2.s7] = v__3.s7; int8 v__4 = (int8)((1)+(8*0), (1)+(8*1), (1)+(8*2), (1)+(8*3), (1)+(8*4), (1)+(8*5), (1)+(8*6), (1)+(8*7)); float8 v__5 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__4.s0],NT_matmul_intermediate_pad_rf_local[v__4.s1],NT_matmul_intermediate_pad_rf_local[v__4.s2],NT_matmul_intermediate_pad_rf_local[v__4.s3],NT_matmul_intermediate_pad_rf_local[v__4.s4],NT_matmul_intermediate_pad_rf_local[v__4.s5],NT_matmul_intermediate_pad_rf_local[v__4.s6],NT_matmul_intermediate_pad_rf_local[v__4.s7])) + ((convert_float8(vload8(0, lv1 + ((((convert_int(get_group_id(1))) * 32768) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, dequantize_local + 8)))); NT_matmul_intermediate_pad_rf_local[v__4.s0] = v__5.s0; NT_matmul_intermediate_pad_rf_local[v__4.s1] = v__5.s1; NT_matmul_intermediate_pad_rf_local[v__4.s2] = v__5.s2; NT_matmul_intermediate_pad_rf_local[v__4.s3] = v__5.s3; NT_matmul_intermediate_pad_rf_local[v__4.s4] = v__5.s4; NT_matmul_intermediate_pad_rf_local[v__4.s5] = v__5.s5; NT_matmul_intermediate_pad_rf_local[v__4.s6] = v__5.s6; NT_matmul_intermediate_pad_rf_local[v__4.s7] = v__5.s7; } if ((((convert_int(get_group_id(1))) * 4) + 1) < batch_size) { int8 v__6 = (int8)((2)+(8*0), (2)+(8*1), (2)+(8*2), (2)+(8*3), (2)+(8*4), (2)+(8*5), (2)+(8*6), (2)+(8*7)); float8 v__7 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__6.s0],NT_matmul_intermediate_pad_rf_local[v__6.s1],NT_matmul_intermediate_pad_rf_local[v__6.s2],NT_matmul_intermediate_pad_rf_local[v__6.s3],NT_matmul_intermediate_pad_rf_local[v__6.s4],NT_matmul_intermediate_pad_rf_local[v__6.s5],NT_matmul_intermediate_pad_rf_local[v__6.s6],NT_matmul_intermediate_pad_rf_local[v__6.s7])) + ((convert_float8(vload8(0, lv1 + (((((convert_int(get_group_id(1))) * 32768) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 8192)))) * (convert_float8(vload8(0, dequantize_local + 0)))); NT_matmul_intermediate_pad_rf_local[v__6.s0] = v__7.s0; NT_matmul_intermediate_pad_rf_local[v__6.s1] = v__7.s1; NT_matmul_intermediate_pad_rf_local[v__6.s2] = v__7.s2; NT_matmul_intermediate_pad_rf_local[v__6.s3] = v__7.s3; NT_matmul_intermediate_pad_rf_local[v__6.s4] = v__7.s4; NT_matmul_intermediate_pad_rf_local[v__6.s5] = v__7.s5; NT_matmul_intermediate_pad_rf_local[v__6.s6] = v__7.s6; NT_matmul_intermediate_pad_rf_local[v__6.s7] = v__7.s7; int8 v__8 = (int8)((3)+(8*0), (3)+(8*1), (3)+(8*2), (3)+(8*3), (3)+(8*4), (3)+(8*5), (3)+(8*6), (3)+(8*7)); float8 v__9 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__8.s0],NT_matmul_intermediate_pad_rf_local[v__8.s1],NT_matmul_intermediate_pad_rf_local[v__8.s2],NT_matmul_intermediate_pad_rf_local[v__8.s3],NT_matmul_intermediate_pad_rf_local[v__8.s4],NT_matmul_intermediate_pad_rf_local[v__8.s5],NT_matmul_intermediate_pad_rf_local[v__8.s6],NT_matmul_intermediate_pad_rf_local[v__8.s7])) + ((convert_float8(vload8(0, lv1 + (((((convert_int(get_group_id(1))) * 32768) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 8192)))) * (convert_float8(vload8(0, dequantize_local + 8)))); NT_matmul_intermediate_pad_rf_local[v__8.s0] = v__9.s0; NT_matmul_intermediate_pad_rf_local[v__8.s1] = v__9.s1; NT_matmul_intermediate_pad_rf_local[v__8.s2] = v__9.s2; NT_matmul_intermediate_pad_rf_local[v__8.s3] = v__9.s3; NT_matmul_intermediate_pad_rf_local[v__8.s4] = v__9.s4; NT_matmul_intermediate_pad_rf_local[v__8.s5] = v__9.s5; NT_matmul_intermediate_pad_rf_local[v__8.s6] = v__9.s6; NT_matmul_intermediate_pad_rf_local[v__8.s7] = v__9.s7; } if ((((convert_int(get_group_id(1))) * 4) + 2) < batch_size) { int8 v__10 = (int8)((4)+(8*0), (4)+(8*1), (4)+(8*2), (4)+(8*3), (4)+(8*4), (4)+(8*5), (4)+(8*6), (4)+(8*7)); float8 v__11 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__10.s0],NT_matmul_intermediate_pad_rf_local[v__10.s1],NT_matmul_intermediate_pad_rf_local[v__10.s2],NT_matmul_intermediate_pad_rf_local[v__10.s3],NT_matmul_intermediate_pad_rf_local[v__10.s4],NT_matmul_intermediate_pad_rf_local[v__10.s5],NT_matmul_intermediate_pad_rf_local[v__10.s6],NT_matmul_intermediate_pad_rf_local[v__10.s7])) + ((convert_float8(vload8(0, lv1 + (((((convert_int(get_group_id(1))) * 32768) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 16384)))) * (convert_float8(vload8(0, dequantize_local + 0)))); NT_matmul_intermediate_pad_rf_local[v__10.s0] = v__11.s0; NT_matmul_intermediate_pad_rf_local[v__10.s1] = v__11.s1; NT_matmul_intermediate_pad_rf_local[v__10.s2] = v__11.s2; NT_matmul_intermediate_pad_rf_local[v__10.s3] = v__11.s3; NT_matmul_intermediate_pad_rf_local[v__10.s4] = v__11.s4; NT_matmul_intermediate_pad_rf_local[v__10.s5] = v__11.s5; NT_matmul_intermediate_pad_rf_local[v__10.s6] = v__11.s6; NT_matmul_intermediate_pad_rf_local[v__10.s7] = v__11.s7; int8 v__12 = (int8)((5)+(8*0), (5)+(8*1), (5)+(8*2), (5)+(8*3), (5)+(8*4), (5)+(8*5), (5)+(8*6), (5)+(8*7)); float8 v__13 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__12.s0],NT_matmul_intermediate_pad_rf_local[v__12.s1],NT_matmul_intermediate_pad_rf_local[v__12.s2],NT_matmul_intermediate_pad_rf_local[v__12.s3],NT_matmul_intermediate_pad_rf_local[v__12.s4],NT_matmul_intermediate_pad_rf_local[v__12.s5],NT_matmul_intermediate_pad_rf_local[v__12.s6],NT_matmul_intermediate_pad_rf_local[v__12.s7])) + ((convert_float8(vload8(0, lv1 + (((((convert_int(get_group_id(1))) * 32768) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 16384)))) * (convert_float8(vload8(0, dequantize_local + 8)))); NT_matmul_intermediate_pad_rf_local[v__12.s0] = v__13.s0; NT_matmul_intermediate_pad_rf_local[v__12.s1] = v__13.s1; NT_matmul_intermediate_pad_rf_local[v__12.s2] = v__13.s2; NT_matmul_intermediate_pad_rf_local[v__12.s3] = v__13.s3; NT_matmul_intermediate_pad_rf_local[v__12.s4] = v__13.s4; NT_matmul_intermediate_pad_rf_local[v__12.s5] = v__13.s5; NT_matmul_intermediate_pad_rf_local[v__12.s6] = v__13.s6; NT_matmul_intermediate_pad_rf_local[v__12.s7] = v__13.s7; } if ((((convert_int(get_group_id(1))) * 4) + 3) < batch_size) { int8 v__14 = (int8)((6)+(8*0), (6)+(8*1), (6)+(8*2), (6)+(8*3), (6)+(8*4), (6)+(8*5), (6)+(8*6), (6)+(8*7)); float8 v__15 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__14.s0],NT_matmul_intermediate_pad_rf_local[v__14.s1],NT_matmul_intermediate_pad_rf_local[v__14.s2],NT_matmul_intermediate_pad_rf_local[v__14.s3],NT_matmul_intermediate_pad_rf_local[v__14.s4],NT_matmul_intermediate_pad_rf_local[v__14.s5],NT_matmul_intermediate_pad_rf_local[v__14.s6],NT_matmul_intermediate_pad_rf_local[v__14.s7])) + ((convert_float8(vload8(0, lv1 + (((((convert_int(get_group_id(1))) * 32768) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 24576)))) * (convert_float8(vload8(0, dequantize_local + 0)))); NT_matmul_intermediate_pad_rf_local[v__14.s0] = v__15.s0; NT_matmul_intermediate_pad_rf_local[v__14.s1] = v__15.s1; NT_matmul_intermediate_pad_rf_local[v__14.s2] = v__15.s2; NT_matmul_intermediate_pad_rf_local[v__14.s3] = v__15.s3; NT_matmul_intermediate_pad_rf_local[v__14.s4] = v__15.s4; NT_matmul_intermediate_pad_rf_local[v__14.s5] = v__15.s5; NT_matmul_intermediate_pad_rf_local[v__14.s6] = v__15.s6; NT_matmul_intermediate_pad_rf_local[v__14.s7] = v__15.s7; int8 v__16 = (int8)((7)+(8*0), (7)+(8*1), (7)+(8*2), (7)+(8*3), (7)+(8*4), (7)+(8*5), (7)+(8*6), (7)+(8*7)); float8 v__17 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__16.s0],NT_matmul_intermediate_pad_rf_local[v__16.s1],NT_matmul_intermediate_pad_rf_local[v__16.s2],NT_matmul_intermediate_pad_rf_local[v__16.s3],NT_matmul_intermediate_pad_rf_local[v__16.s4],NT_matmul_intermediate_pad_rf_local[v__16.s5],NT_matmul_intermediate_pad_rf_local[v__16.s6],NT_matmul_intermediate_pad_rf_local[v__16.s7])) + ((convert_float8(vload8(0, lv1 + (((((convert_int(get_group_id(1))) * 32768) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 24576)))) * (convert_float8(vload8(0, dequantize_local + 8)))); NT_matmul_intermediate_pad_rf_local[v__16.s0] = v__17.s0; NT_matmul_intermediate_pad_rf_local[v__16.s1] = v__17.s1; NT_matmul_intermediate_pad_rf_local[v__16.s2] = v__17.s2; NT_matmul_intermediate_pad_rf_local[v__16.s3] = v__17.s3; NT_matmul_intermediate_pad_rf_local[v__16.s4] = v__17.s4; NT_matmul_intermediate_pad_rf_local[v__16.s5] = v__17.s5; NT_matmul_intermediate_pad_rf_local[v__16.s6] = v__17.s6; NT_matmul_intermediate_pad_rf_local[v__16.s7] = v__17.s7; } } for (int ax3_fused_2_0 = 0; ax3_fused_2_0 < 1; ++ax3_fused_2_0) { for (int ax2 = 0; ax2 < 4; ++ax2) { vstore2(((float2)(0.000000e+00f, 0.000000e+00f)), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + (ax2 * 2))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 8))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 16))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 24))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 32))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 40))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 48))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 56))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); } } for (int ax2_fused_2 = 0; ax2_fused_2 < 2; ++ax2_fused_2) { for (int ax1 = 0; ax1 < 4; ++ax1) { barrier(CLK_LOCAL_MEM_FENCE); ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_intermediate_pad_rf_local_1[((ax1 * 2) + ax2_fused_2)]; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 16) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 8) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 4) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 2) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); } barrier(CLK_LOCAL_MEM_FENCE); NT_matmul_intermediate_pad_local[((ax1 * 2) + ax2_fused_2)] = ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) * 32)]; } } for (int ax0 = 0; ax0 < 4; ++ax0) { for (int ax1_fused_2 = 0; ax1_fused_2 < 2; ++ax1_fused_2) { if (((bool)(((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)(((bool)(((convert_int(get_group_id(1))) - ((batch_size + 3) >> 2)) < 0)) || ((bool)((convert_int(get_group_id(1))) == 0)))))) && ((bool)((((convert_int(get_group_id(1))) * 4) + ax0) < batch_size))) { compute[((((((convert_int(get_group_id(1))) * 8192) + (ax0 * 2048)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] = (convert_half((NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias4[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]))); } } } } // Function: gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50) { float NT_matmul_rf_local[8]; uint gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2_local[1]; float NT_matmul_rf_local_1[1]; __local float red_buf0[64]; vstore8(((float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)), 0, NT_matmul_rf_local + 0); for (int ax1_0_fused_ax1_1_fused_0 = 0; ax1_0_fused_ax1_1_fused_0 < 8; ++ax1_0_fused_ax1_1_fused_0) { gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2_local[0] = gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2[(((((convert_int(get_group_id(0))) * 512) + ((convert_int(get_local_id(0))) * 256)) + (ax1_0_fused_ax1_1_fused_0 * 32)) + (convert_int(get_local_id(1))))]; vstore8((vload8(0, NT_matmul_rf_local + 0) + ((convert_float8(vload8(0, layer_norm50 + ((ax1_0_fused_ax1_1_fused_0 * 256) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8((((convert_half8(((((uint8)(gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2_local[0], gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2_local[0], gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2_local[0], gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2_local[0], gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2_local[0], gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2_local[0], gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2_local[0], gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2_local[0])) >> ((uint8)(((uint)0)+((uint)4*0), ((uint)0)+((uint)4*1), ((uint)0)+((uint)4*2), ((uint)0)+((uint)4*3), ((uint)0)+((uint)4*4), ((uint)0)+((uint)4*5), ((uint)0)+((uint)4*6), ((uint)0)+((uint)4*7)))) & ((uint8)((uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15))))) - ((half8)((half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f))) * ((half8)(gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2[(((((convert_int(get_group_id(0))) * 128) + ((convert_int(get_local_id(0))) * 64)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2[(((((convert_int(get_group_id(0))) * 128) + ((convert_int(get_local_id(0))) * 64)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2[(((((convert_int(get_group_id(0))) * 128) + ((convert_int(get_local_id(0))) * 64)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2[(((((convert_int(get_group_id(0))) * 128) + ((convert_int(get_local_id(0))) * 64)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2[(((((convert_int(get_group_id(0))) * 128) + ((convert_int(get_local_id(0))) * 64)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2[(((((convert_int(get_group_id(0))) * 128) + ((convert_int(get_local_id(0))) * 64)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2[(((((convert_int(get_group_id(0))) * 128) + ((convert_int(get_local_id(0))) * 64)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2[(((((convert_int(get_group_id(0))) * 128) + ((convert_int(get_local_id(0))) * 64)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]))))))), 0, NT_matmul_rf_local + 0); } for (int ax2_fused_2_0 = 0; ax2_fused_2_0 < 1; ++ax2_fused_2_0) { NT_matmul_rf_local_1[0] = 0.000000e+00f; NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[0]); NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[1]); NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[2]); NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[3]); NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[4]); NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[5]); NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[6]); NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[7]); } barrier(CLK_LOCAL_MEM_FENCE); ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_rf_local_1[0]; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 16) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 8) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 4) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 2) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) == 0) { compute[(((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0))))] = (convert_half(((((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) * 32)] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias2[(((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0))))]) * (5.000000e-01f + (erf(((((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) * 32)] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias2[(((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0))))]) * 7.071068e-01f)) * 5.000000e-01f))))); } } // Function: gpt_neox_q4f16_1_reshape4_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len) { T_reshape[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] = add288[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]; } // Function: gpt_neox_q4f16_1_reshape2_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size) { T_reshape[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] = lv366[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]; } // Function: gpt_neox_q4f16_1_reshape_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size) { T_reshape[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] = add432[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]; } // Function: gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen) { int position = position_map[(((convert_int(get_group_id(0))) >> 3) + position_map_elem_offset)]; k_data[((((layer_id * seqlen) * 2048) + ((convert_int(get_group_id(0))) * 256)) + (convert_int(get_local_id(0))))] = pages[((((((((((position / page_size) + ((position % page_size) >> 31)) * 32) + (((convert_int(get_group_id(0))) & 7) * 2)) + ((convert_int(get_local_id(0))) >> 7)) * page_size) * 128) + (((position % page_size) + (page_size & ((position % page_size) >> 31))) * 128)) + pages_elem_offset) + ((convert_int(get_local_id(0))) & 127))]; v_data[((((layer_id * seqlen) * 2048) + ((convert_int(get_group_id(0))) * 256)) + (convert_int(get_local_id(0))))] = pages[(((((((((((position / page_size) + ((position % page_size) >> 31)) * 32) + (((convert_int(get_group_id(0))) & 7) * 2)) + ((convert_int(get_local_id(0))) >> 7)) + 16) * page_size) * 128) + (((position % page_size) + (page_size & ((position % page_size) >> 31))) * 128)) + pages_elem_offset) + ((convert_int(get_local_id(0))) & 127))]; } // Function: gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size) { if ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) < num_token) { logits[((pos2seq_id[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] * vocab_size) + token_ids[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))])] = (logits[((pos2seq_id[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] * vocab_size) + token_ids[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))])] + logit_bias[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]); } } // Function: gpt_neox_q4f16_1_take_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len) { T_take[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] = layer_norm146[(((logit_positions[((convert_int(get_group_id(0))) >> 3)] * 2048) + (((convert_int(get_group_id(0))) & 7) * 256)) + (convert_int(get_local_id(0))))]; } // Function: gpt_neox_q4f16_1_layer_norm1_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len) { float in_thread_input_embeds_red_temp_v0_shared[1]; float in_thread_input_embeds_red_temp_v1_shared[1]; __local float red_buf0[256]; __local float red_buf1[256]; __local float input_embeds_red_temp_v0_shared[1]; __local float input_embeds_red_temp_v1_shared[1]; in_thread_input_embeds_red_temp_v0_shared[0] = 0.000000e+00f; in_thread_input_embeds_red_temp_v1_shared[0] = 0.000000e+00f; float v_input_embeds_red_temp_v0 = (in_thread_input_embeds_red_temp_v0_shared[0] + (convert_float(input_embeds[(((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0))))]))); float v_input_embeds_red_temp_v1 = (in_thread_input_embeds_red_temp_v1_shared[0] + ((convert_float(input_embeds[(((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0))))])) * (convert_float(input_embeds[(((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0))))])))); in_thread_input_embeds_red_temp_v0_shared[0] = v_input_embeds_red_temp_v0; in_thread_input_embeds_red_temp_v1_shared[0] = v_input_embeds_red_temp_v1; float v_input_embeds_red_temp_v0_1 = (in_thread_input_embeds_red_temp_v0_shared[0] + (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 256)]))); float v_input_embeds_red_temp_v1_1 = (in_thread_input_embeds_red_temp_v1_shared[0] + ((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 256)])) * (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 256)])))); in_thread_input_embeds_red_temp_v0_shared[0] = v_input_embeds_red_temp_v0_1; in_thread_input_embeds_red_temp_v1_shared[0] = v_input_embeds_red_temp_v1_1; float v_input_embeds_red_temp_v0_2 = (in_thread_input_embeds_red_temp_v0_shared[0] + (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 512)]))); float v_input_embeds_red_temp_v1_2 = (in_thread_input_embeds_red_temp_v1_shared[0] + ((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 512)])) * (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 512)])))); in_thread_input_embeds_red_temp_v0_shared[0] = v_input_embeds_red_temp_v0_2; in_thread_input_embeds_red_temp_v1_shared[0] = v_input_embeds_red_temp_v1_2; float v_input_embeds_red_temp_v0_3 = (in_thread_input_embeds_red_temp_v0_shared[0] + (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 768)]))); float v_input_embeds_red_temp_v1_3 = (in_thread_input_embeds_red_temp_v1_shared[0] + ((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 768)])) * (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 768)])))); in_thread_input_embeds_red_temp_v0_shared[0] = v_input_embeds_red_temp_v0_3; in_thread_input_embeds_red_temp_v1_shared[0] = v_input_embeds_red_temp_v1_3; float v_input_embeds_red_temp_v0_4 = (in_thread_input_embeds_red_temp_v0_shared[0] + (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1024)]))); float v_input_embeds_red_temp_v1_4 = (in_thread_input_embeds_red_temp_v1_shared[0] + ((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1024)])) * (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1024)])))); in_thread_input_embeds_red_temp_v0_shared[0] = v_input_embeds_red_temp_v0_4; in_thread_input_embeds_red_temp_v1_shared[0] = v_input_embeds_red_temp_v1_4; float v_input_embeds_red_temp_v0_5 = (in_thread_input_embeds_red_temp_v0_shared[0] + (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1280)]))); float v_input_embeds_red_temp_v1_5 = (in_thread_input_embeds_red_temp_v1_shared[0] + ((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1280)])) * (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1280)])))); in_thread_input_embeds_red_temp_v0_shared[0] = v_input_embeds_red_temp_v0_5; in_thread_input_embeds_red_temp_v1_shared[0] = v_input_embeds_red_temp_v1_5; float v_input_embeds_red_temp_v0_6 = (in_thread_input_embeds_red_temp_v0_shared[0] + (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1536)]))); float v_input_embeds_red_temp_v1_6 = (in_thread_input_embeds_red_temp_v1_shared[0] + ((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1536)])) * (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1536)])))); in_thread_input_embeds_red_temp_v0_shared[0] = v_input_embeds_red_temp_v0_6; in_thread_input_embeds_red_temp_v1_shared[0] = v_input_embeds_red_temp_v1_6; float v_input_embeds_red_temp_v0_7 = (in_thread_input_embeds_red_temp_v0_shared[0] + (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1792)]))); float v_input_embeds_red_temp_v1_7 = (in_thread_input_embeds_red_temp_v1_shared[0] + ((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1792)])) * (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1792)])))); in_thread_input_embeds_red_temp_v0_shared[0] = v_input_embeds_red_temp_v0_7; in_thread_input_embeds_red_temp_v1_shared[0] = v_input_embeds_red_temp_v1_7; barrier(CLK_LOCAL_MEM_FENCE); ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = in_thread_input_embeds_red_temp_v0_shared[0]; ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = in_thread_input_embeds_red_temp_v1_shared[0]; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 128) { ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 128)]); ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 128)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 64) { ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 64)]); ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 64)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 32) { ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 32)]); ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 32)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 16) { ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 16)]); ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 16)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 8) { ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 8)]); ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 8)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 4) { ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 4)]); ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 4)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 2) { ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 2)]); ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 2)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 1) { ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 1)]); ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 1)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) == 0) { input_embeds_red_temp_v0_shared[0] = ((volatile __local float*)red_buf0)[0]; input_embeds_red_temp_v1_shared[0] = ((volatile __local float*)red_buf1)[0]; } barrier(CLK_LOCAL_MEM_FENCE); T_layer_norm[(((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0))))] = (((convert_half((((convert_float(input_embeds[(((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0))))])) - (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embeds_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embeds_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight3[(convert_int(get_local_id(0)))]) + gpt_neox_layers_0_input_layernorm_bias3[(convert_int(get_local_id(0)))]); T_layer_norm[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 256)] = (((convert_half((((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 256)])) - (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embeds_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embeds_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight3[((convert_int(get_local_id(0))) + 256)]) + gpt_neox_layers_0_input_layernorm_bias3[((convert_int(get_local_id(0))) + 256)]); T_layer_norm[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 512)] = (((convert_half((((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 512)])) - (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embeds_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embeds_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight3[((convert_int(get_local_id(0))) + 512)]) + gpt_neox_layers_0_input_layernorm_bias3[((convert_int(get_local_id(0))) + 512)]); T_layer_norm[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 768)] = (((convert_half((((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 768)])) - (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embeds_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embeds_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight3[((convert_int(get_local_id(0))) + 768)]) + gpt_neox_layers_0_input_layernorm_bias3[((convert_int(get_local_id(0))) + 768)]); T_layer_norm[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1024)] = (((convert_half((((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1024)])) - (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embeds_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embeds_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight3[((convert_int(get_local_id(0))) + 1024)]) + gpt_neox_layers_0_input_layernorm_bias3[((convert_int(get_local_id(0))) + 1024)]); T_layer_norm[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1280)] = (((convert_half((((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1280)])) - (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embeds_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embeds_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight3[((convert_int(get_local_id(0))) + 1280)]) + gpt_neox_layers_0_input_layernorm_bias3[((convert_int(get_local_id(0))) + 1280)]); T_layer_norm[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1536)] = (((convert_half((((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1536)])) - (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embeds_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embeds_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight3[((convert_int(get_local_id(0))) + 1536)]) + gpt_neox_layers_0_input_layernorm_bias3[((convert_int(get_local_id(0))) + 1536)]); T_layer_norm[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1792)] = (((convert_half((((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1792)])) - (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embeds_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embeds_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight3[((convert_int(get_local_id(0))) + 1792)]) + gpt_neox_layers_0_input_layernorm_bias3[((convert_int(get_local_id(0))) + 1792)]); } // Function: gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2 #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len) { half8 NT_matmul_intermediate_reindex_pad_local[2]; for (int var = 0; var < 1; ++var) { NT_matmul_intermediate_reindex_pad_local[0] = ((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)); NT_matmul_intermediate_reindex_pad_local[1] = ((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)); for (int ax3_0 = 0; ax3_0 < 128; ++ax3_0) { for (int ax3_1 = 0; ax3_1 < 16; ++ax3_1) { uint cse_var_7 = (convert_uint(((ax3_1 & 7) * 4))); half condval; if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < seq_len)) { condval = layer_norm98[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; } else { condval = (half)0.000000e+00f; } int8 v_ = (int8)(((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*0), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*1), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*2), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*3), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*4), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*5), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*6), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*7)); int8 v__1 = (int8)((((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*0), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*1), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*2), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*3), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*4), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*5), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*6), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*7)); NT_matmul_intermediate_reindex_pad_local[0] = (NT_matmul_intermediate_reindex_pad_local[0] + (((half8)(condval, condval, condval, condval, condval, condval, condval, condval)) * (((convert_half8(((((uint8)(gpt_neox_layers_0_attention_query_key_value_q_weight3[v_.s0],gpt_neox_layers_0_attention_query_key_value_q_weight3[v_.s1],gpt_neox_layers_0_attention_query_key_value_q_weight3[v_.s2],gpt_neox_layers_0_attention_query_key_value_q_weight3[v_.s3],gpt_neox_layers_0_attention_query_key_value_q_weight3[v_.s4],gpt_neox_layers_0_attention_query_key_value_q_weight3[v_.s5],gpt_neox_layers_0_attention_query_key_value_q_weight3[v_.s6],gpt_neox_layers_0_attention_query_key_value_q_weight3[v_.s7])) >> ((uint8)(cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7))) & ((uint8)((uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15))))) - ((half8)((half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f))) * ((half8)(gpt_neox_layers_0_attention_query_key_value_q_scale3[v__1.s0],gpt_neox_layers_0_attention_query_key_value_q_scale3[v__1.s1],gpt_neox_layers_0_attention_query_key_value_q_scale3[v__1.s2],gpt_neox_layers_0_attention_query_key_value_q_scale3[v__1.s3],gpt_neox_layers_0_attention_query_key_value_q_scale3[v__1.s4],gpt_neox_layers_0_attention_query_key_value_q_scale3[v__1.s5],gpt_neox_layers_0_attention_query_key_value_q_scale3[v__1.s6],gpt_neox_layers_0_attention_query_key_value_q_scale3[v__1.s7]))))); half condval_1; if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < seq_len)) { condval_1 = layer_norm98[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; } else { condval_1 = (half)0.000000e+00f; } int8 v__2 = (int8)(((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*0), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*1), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*2), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*3), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*4), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*5), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*6), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*7)); int8 v__3 = (int8)((((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*0), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*1), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*2), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*3), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*4), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*5), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*6), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*7)); NT_matmul_intermediate_reindex_pad_local[1] = (NT_matmul_intermediate_reindex_pad_local[1] + (((half8)(condval_1, condval_1, condval_1, condval_1, condval_1, condval_1, condval_1, condval_1)) * (((convert_half8(((((uint8)(gpt_neox_layers_0_attention_query_key_value_q_weight3[v__2.s0],gpt_neox_layers_0_attention_query_key_value_q_weight3[v__2.s1],gpt_neox_layers_0_attention_query_key_value_q_weight3[v__2.s2],gpt_neox_layers_0_attention_query_key_value_q_weight3[v__2.s3],gpt_neox_layers_0_attention_query_key_value_q_weight3[v__2.s4],gpt_neox_layers_0_attention_query_key_value_q_weight3[v__2.s5],gpt_neox_layers_0_attention_query_key_value_q_weight3[v__2.s6],gpt_neox_layers_0_attention_query_key_value_q_weight3[v__2.s7])) >> ((uint8)(cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7))) & ((uint8)((uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15))))) - ((half8)((half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f))) * ((half8)(gpt_neox_layers_0_attention_query_key_value_q_scale3[v__3.s0],gpt_neox_layers_0_attention_query_key_value_q_scale3[v__3.s1],gpt_neox_layers_0_attention_query_key_value_q_scale3[v__3.s2],gpt_neox_layers_0_attention_query_key_value_q_scale3[v__3.s3],gpt_neox_layers_0_attention_query_key_value_q_scale3[v__3.s4],gpt_neox_layers_0_attention_query_key_value_q_scale3[v__3.s5],gpt_neox_layers_0_attention_query_key_value_q_scale3[v__3.s6],gpt_neox_layers_0_attention_query_key_value_q_scale3[v__3.s7]))))); } } if ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < seq_len) { vstore8((NT_matmul_intermediate_reindex_pad_local[0] + vload8(0, gpt_neox_layers_0_attention_query_key_value_bias3 + (((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)))), 0, T_add + (((((convert_int(get_group_id(1))) * 49152) + ((convert_int(get_local_id(1))) * 12288)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8))); } if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < seq_len) { vstore8((NT_matmul_intermediate_reindex_pad_local[1] + vload8(0, gpt_neox_layers_0_attention_query_key_value_bias3 + (((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)))), 0, T_add + ((((((convert_int(get_group_id(1))) * 49152) + ((convert_int(get_local_id(1))) * 12288)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 6144)); } } } // Function: gpt_neox_q4f16_1_gather_probs_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n) { if ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) < (batch_size * n)) { dst[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] = src[((indices[(((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % (n * batch_size)) / n)] * n) + ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % n))]; } } // Function: gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144) { T_reshape[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] = add144[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]; } // Function: gpt_neox_q4f16_1_fused_rope_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len) { if (((convert_int(get_group_id(0))) % 24) < 8) { half condval; if ((((bool)(0 < apply_rope)) && ((bool)(((convert_int(get_local_id(0))) & 127) < 32)))) { float freq = ((convert_float(position_map[(((convert_int(get_group_id(0))) / 24) + position_map_elem_offset)])) / pow(1.000000e+04f, ((convert_float((((convert_int(get_local_id(0))) & 15) * 2))) * 3.125000e-02f))); half condval_1; if ((((convert_int(get_local_id(0))) & 127) < 16)) { condval_1 = (qkv[((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) + 16)] * (half)-1.000000e+00f); } else { condval_1 = qkv[((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) - 16)]; } condval = (convert_half(((cos(freq) * (convert_float(qkv[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]))) + (sin(freq) * (convert_float(condval_1)))))); } else { condval = qkv[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]; } q[(((((convert_int(get_group_id(0))) / 24) * 2048) + (((convert_int(get_group_id(0))) % 24) * 256)) + (convert_int(get_local_id(0))))] = condval; } else { if (((convert_int(get_group_id(0))) % 24) < 16) { half condval_2; if ((((bool)(0 < apply_rope)) && ((bool)(((convert_int(get_local_id(0))) & 127) < 32)))) { float freq_1 = ((convert_float(position_map[(((convert_int(get_group_id(0))) / 24) + position_map_elem_offset)])) / pow(1.000000e+04f, ((convert_float((((convert_int(get_local_id(0))) & 15) * 2))) * 3.125000e-02f))); half condval_3; if ((((convert_int(get_local_id(0))) & 127) < 16)) { condval_3 = (qkv[((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) + 16)] * (half)-1.000000e+00f); } else { condval_3 = qkv[((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) - 16)]; } condval_2 = (convert_half(((cos(freq_1) * (convert_float(qkv[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]))) + (sin(freq_1) * (convert_float(condval_3)))))); } else { condval_2 = qkv[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]; } k[((((((convert_int(get_group_id(0))) / 24) * 2048) + (((convert_int(get_group_id(0))) % 24) * 256)) + (convert_int(get_local_id(0)))) - 2048)] = condval_2; } else { v[((((((convert_int(get_group_id(0))) / 24) * 2048) + (((convert_int(get_group_id(0))) % 24) * 256)) + (convert_int(get_local_id(0)))) - 4096)] = qkv[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]; } } } // Function: gpt_neox_q4f16_1_reshape3_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size) { T_reshape[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] = reshape290[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]; } // Function: gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2 #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len) { half8 NT_matmul_intermediate_reindex_pad_local[2]; for (int var = 0; var < 1; ++var) { NT_matmul_intermediate_reindex_pad_local[0] = ((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)); NT_matmul_intermediate_reindex_pad_local[1] = ((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)); for (int ax3_0 = 0; ax3_0 < 128; ++ax3_0) { for (int ax3_1 = 0; ax3_1 < 16; ++ax3_1) { uint cse_var_7 = (convert_uint(((ax3_1 & 7) * 4))); half condval; if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < seq_len)) { condval = reshape195[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; } else { condval = (half)0.000000e+00f; } int8 v_ = (int8)(((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*0), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*1), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*2), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*3), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*4), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*5), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*6), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*7)); int8 v__1 = (int8)((((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*0), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*1), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*2), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*3), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*4), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*5), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*6), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*7)); NT_matmul_intermediate_reindex_pad_local[0] = (NT_matmul_intermediate_reindex_pad_local[0] + (((half8)(condval, condval, condval, condval, condval, condval, condval, condval)) * (((convert_half8(((((uint8)(gpt_neox_layers_0_attention_dense_q_weight3[v_.s0],gpt_neox_layers_0_attention_dense_q_weight3[v_.s1],gpt_neox_layers_0_attention_dense_q_weight3[v_.s2],gpt_neox_layers_0_attention_dense_q_weight3[v_.s3],gpt_neox_layers_0_attention_dense_q_weight3[v_.s4],gpt_neox_layers_0_attention_dense_q_weight3[v_.s5],gpt_neox_layers_0_attention_dense_q_weight3[v_.s6],gpt_neox_layers_0_attention_dense_q_weight3[v_.s7])) >> ((uint8)(cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7))) & ((uint8)((uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15))))) - ((half8)((half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f))) * ((half8)(gpt_neox_layers_0_attention_dense_q_scale3[v__1.s0],gpt_neox_layers_0_attention_dense_q_scale3[v__1.s1],gpt_neox_layers_0_attention_dense_q_scale3[v__1.s2],gpt_neox_layers_0_attention_dense_q_scale3[v__1.s3],gpt_neox_layers_0_attention_dense_q_scale3[v__1.s4],gpt_neox_layers_0_attention_dense_q_scale3[v__1.s5],gpt_neox_layers_0_attention_dense_q_scale3[v__1.s6],gpt_neox_layers_0_attention_dense_q_scale3[v__1.s7]))))); half condval_1; if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < seq_len)) { condval_1 = reshape195[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; } else { condval_1 = (half)0.000000e+00f; } int8 v__2 = (int8)(((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*0), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*1), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*2), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*3), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*4), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*5), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*6), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*7)); int8 v__3 = (int8)((((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*0), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*1), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*2), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*3), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*4), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*5), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*6), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*7)); NT_matmul_intermediate_reindex_pad_local[1] = (NT_matmul_intermediate_reindex_pad_local[1] + (((half8)(condval_1, condval_1, condval_1, condval_1, condval_1, condval_1, condval_1, condval_1)) * (((convert_half8(((((uint8)(gpt_neox_layers_0_attention_dense_q_weight3[v__2.s0],gpt_neox_layers_0_attention_dense_q_weight3[v__2.s1],gpt_neox_layers_0_attention_dense_q_weight3[v__2.s2],gpt_neox_layers_0_attention_dense_q_weight3[v__2.s3],gpt_neox_layers_0_attention_dense_q_weight3[v__2.s4],gpt_neox_layers_0_attention_dense_q_weight3[v__2.s5],gpt_neox_layers_0_attention_dense_q_weight3[v__2.s6],gpt_neox_layers_0_attention_dense_q_weight3[v__2.s7])) >> ((uint8)(cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7))) & ((uint8)((uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15))))) - ((half8)((half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f))) * ((half8)(gpt_neox_layers_0_attention_dense_q_scale3[v__3.s0],gpt_neox_layers_0_attention_dense_q_scale3[v__3.s1],gpt_neox_layers_0_attention_dense_q_scale3[v__3.s2],gpt_neox_layers_0_attention_dense_q_scale3[v__3.s3],gpt_neox_layers_0_attention_dense_q_scale3[v__3.s4],gpt_neox_layers_0_attention_dense_q_scale3[v__3.s5],gpt_neox_layers_0_attention_dense_q_scale3[v__3.s6],gpt_neox_layers_0_attention_dense_q_scale3[v__3.s7]))))); } } if ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < seq_len) { vstore8(((vload8(0, lv74 + (((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8))) + (NT_matmul_intermediate_reindex_pad_local[0] + vload8(0, gpt_neox_layers_0_attention_dense_bias3 + (((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8))))) + vload8(0, input_embeds + (((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)))), 0, T_add + (((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8))); } if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < seq_len) { vstore8(((vload8(0, lv74 + ((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 2048)) + (NT_matmul_intermediate_reindex_pad_local[1] + vload8(0, gpt_neox_layers_0_attention_dense_bias3 + (((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8))))) + vload8(0, input_embeds + ((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 2048))), 0, T_add + ((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 2048)); } } } // Function: gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len) { int tile_id[1]; int batch_idx[1]; int batch_rows[1]; int batch_tiles[1]; int kv_chunk_len[1]; __local float m_smem[32]; __local float d_smem[32]; float O_local[32]; __local half Q_smem[4096]; __local half K_smem[2048]; __local half V_smem[2048]; float S_local[4]; float m_prev[1]; float m_new[1]; float d_new[1]; __local float m_prev_smem[32]; tile_id[0] = (convert_int(get_group_id(0))); batch_idx[0] = 0; batch_rows[0] = (q_indptr[(q_indptr_elem_offset + 1)] - q_indptr[q_indptr_elem_offset]); batch_tiles[0] = ((batch_rows[0] + 31) >> 5); while (1) { if (!(((batch_idx[0] < batch_size)))) { break; } while (1) { if (!((((bool)(batch_tiles[0] <= tile_id[0])) && ((bool)(batch_idx[0] < batch_size))))) { break; } tile_id[0] = (tile_id[0] - batch_tiles[0]); batch_idx[0] = (batch_idx[0] + 1); if (batch_idx[0] < batch_size) { int b_idx = batch_idx[0]; batch_rows[0] = (q_indptr[((b_idx + q_indptr_elem_offset) + 1)] - q_indptr[(b_idx + q_indptr_elem_offset)]); batch_tiles[0] = ((batch_rows[0] + 31) >> 5); } } if ((batch_idx[0] < batch_size)) { int b_idx_1 = batch_idx[0]; int LH_start = (tile_id[0] * 32); int q_indptr_val = q_indptr[(b_idx_1 + q_indptr_elem_offset)]; int cur_page_indptr_begin = page_indptr[(b_idx_1 + page_indptr_elem_offset)]; int cur_page_indptr_end = page_indptr[((b_idx_1 + page_indptr_elem_offset) + 1)]; int condval; if ((cur_page_indptr_begin != cur_page_indptr_end)) { condval = ((((cur_page_indptr_end * 16) + length_info[(b_idx_1 + length_info_elem_offset)]) - (cur_page_indptr_begin * 16)) - 16); } else { condval = 0; } kv_chunk_len[0] = condval; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { m_smem[(convert_int(get_local_id(0)))] = -5.000000e+04f; d_smem[(convert_int(get_local_id(0)))] = 1.000000e+00f; } for (int li_1 = 0; li_1 < 4; ++li_1) { for (int lj_1 = 0; lj_1 < 8; ++lj_1) { O_local[((li_1 * 8) + lj_1)] = 0.000000e+00f; } } barrier(CLK_LOCAL_MEM_FENCE); for (int li_lj_fused_0 = 0; li_lj_fused_0 < 8; ++li_lj_fused_0) { if (((((li_lj_fused_0 * 4) + q_indptr_val) + LH_start) + (convert_int(get_local_id(1)))) < q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)]) { half4 condval_1; if ((rotary_mode == 1)) { float4 freq = (((float4)(((convert_float(q_rope_position[(((((li_lj_fused_0 * 4) + (convert_int(get_local_id(1)))) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[(((((li_lj_fused_0 * 4) + (convert_int(get_local_id(1)))) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[(((((li_lj_fused_0 * 4) + (convert_int(get_local_id(1)))) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[(((((li_lj_fused_0 * 4) + (convert_int(get_local_id(1)))) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale))) / pow(((float4)(rope_theta, rope_theta, rope_theta, rope_theta)), ((convert_float4(((int4)(((((convert_int(get_local_id(0))) & 15) * 8))+(2*0), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*1), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*2), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*3))))) * ((float4)(7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f))))); half4 condval_2; if (((convert_int(get_local_id(0))) < 16)) { condval_2 = (vload4(0, q + (((((((li_lj_fused_0 * 8192) + ((convert_int(get_local_id(1))) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + 64)) * ((half4)((half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f))); } else { condval_2 = vload4(0, q + (((((((li_lj_fused_0 * 8192) + ((convert_int(get_local_id(1))) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) - 64)); } condval_1 = (convert_half4(((cos(freq) * (convert_float4(vload4(0, q + ((((((li_lj_fused_0 * 8192) + ((convert_int(get_local_id(1))) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)))))) + (sin(freq) * (convert_float4(condval_2)))))); } else { condval_1 = vload4(0, q + ((((((li_lj_fused_0 * 8192) + ((convert_int(get_local_id(1))) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); } vstore4(condval_1, 0, Q_smem + (((li_lj_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); } else { vstore4(((half4)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, Q_smem + (((li_lj_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); } } barrier(CLK_LOCAL_MEM_FENCE); for (int iterator = 0; iterator < ((kv_chunk_len[0] + 15) >> 4); ++iterator) { barrier(CLK_LOCAL_MEM_FENCE); for (int lz_ly_fused_0 = 0; lz_ly_fused_0 < 4; ++lz_ly_fused_0) { if ((((iterator * 16) + (lz_ly_fused_0 * 4)) + (convert_int(get_local_id(1)))) < kv_chunk_len[0]) { int page_no = page_values[((iterator + cur_page_indptr_begin) + page_values_elem_offset)]; half4 condval_3; if ((rotary_mode == 1)) { float4 freq_1 = (((float4)(((convert_float(((((iterator * 16) + (lz_ly_fused_0 * 4)) + k_rope_pos_offset[(b_idx_1 + k_rope_pos_offset_elem_offset)]) + (convert_int(get_local_id(1)))))) * rope_scale), ((convert_float(((((iterator * 16) + (lz_ly_fused_0 * 4)) + k_rope_pos_offset[(b_idx_1 + k_rope_pos_offset_elem_offset)]) + (convert_int(get_local_id(1)))))) * rope_scale), ((convert_float(((((iterator * 16) + (lz_ly_fused_0 * 4)) + k_rope_pos_offset[(b_idx_1 + k_rope_pos_offset_elem_offset)]) + (convert_int(get_local_id(1)))))) * rope_scale), ((convert_float(((((iterator * 16) + (lz_ly_fused_0 * 4)) + k_rope_pos_offset[(b_idx_1 + k_rope_pos_offset_elem_offset)]) + (convert_int(get_local_id(1)))))) * rope_scale))) / pow(((float4)(rope_theta, rope_theta, rope_theta, rope_theta)), ((convert_float4(((int4)(((((convert_int(get_local_id(0))) & 15) * 8))+(2*0), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*1), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*2), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*3))))) * ((float4)(7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f))))); int4 v_ = (int4)((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*0), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*1), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*2), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*3)); half4 condval_4; if (((convert_int(get_local_id(0))) < 16)) { int4 v__1 = (int4)(((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 64))+(1*0), ((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 64))+(1*1), ((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 64))+(1*2), ((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 64))+(1*3)); condval_4 = (((half4)(pages[v__1.s0],pages[v__1.s1],pages[v__1.s2],pages[v__1.s3])) * ((half4)((half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f))); } else { int4 v__2 = (int4)(((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) - 64))+(1*0), ((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) - 64))+(1*1), ((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) - 64))+(1*2), ((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) - 64))+(1*3)); condval_4 = ((half4)(pages[v__2.s0],pages[v__2.s1],pages[v__2.s2],pages[v__2.s3])); } condval_3 = (convert_half4(((cos(freq_1) * (convert_float4(((half4)(pages[v_.s0],pages[v_.s1],pages[v_.s2],pages[v_.s3]))))) + (sin(freq_1) * (convert_float4(condval_4)))))); } else { int4 v__3 = (int4)((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*0), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*1), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*2), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*3)); condval_3 = ((half4)(pages[v__3.s0],pages[v__3.s1],pages[v__3.s2],pages[v__3.s3])); } vstore4(condval_3, 0, K_smem + (((lz_ly_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); } else { vstore4(((half4)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, K_smem + (((lz_ly_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); } } barrier(CLK_LOCAL_MEM_FENCE); for (int lz_ly_fused_0_1 = 0; lz_ly_fused_0_1 < 4; ++lz_ly_fused_0_1) { if ((((iterator * 16) + (lz_ly_fused_0_1 * 4)) + (convert_int(get_local_id(1)))) < kv_chunk_len[0]) { int page_no_1 = page_values[((iterator + cur_page_indptr_begin) + page_values_elem_offset)]; int4 v__4 = (int4)(((((((((page_no_1 * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0_1 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 32768))+(1*0), ((((((((page_no_1 * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0_1 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 32768))+(1*1), ((((((((page_no_1 * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0_1 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 32768))+(1*2), ((((((((page_no_1 * 65536) + ((convert_int(get_group_id(1))) * 2048)) + (lz_ly_fused_0_1 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 32768))+(1*3)); vstore4(((half4)(pages[v__4.s0],pages[v__4.s1],pages[v__4.s2],pages[v__4.s3])), 0, V_smem + (((lz_ly_fused_0_1 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); } else { vstore4(((half4)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, V_smem + (((lz_ly_fused_0_1 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); } } barrier(CLK_LOCAL_MEM_FENCE); for (int li_1_init = 0; li_1_init < 2; ++li_1_init) { for (int lj_1_init = 0; lj_1_init < 2; ++lj_1_init) { S_local[((li_1_init * 2) + lj_1_init)] = 0.000000e+00f; } } for (int lk_0 = 0; lk_0 < 16; ++lk_0) { for (int li_1_1 = 0; li_1_1 < 2; ++li_1_1) { for (int lj_1_1 = 0; lj_1_1 < 2; ++lj_1_1) { for (int lk_1 = 0; lk_1 < 8; ++lk_1) { S_local[((li_1_1 * 2) + lj_1_1)] = (S_local[((li_1_1 * 2) + lj_1_1)] + ((((convert_float(Q_smem[((((((convert_int(get_local_id(1))) * 1024) + (((convert_int(get_local_id(0))) >> 3) * 256)) + (li_1_1 * 128)) + (lk_0 * 8)) + lk_1)])) * (convert_float(K_smem[((((((convert_int(get_local_id(0))) & 7) * 256) + (lj_1_1 * 128)) + (lk_0 * 8)) + lk_1)]))) * attn_score_scaling_factor) * 1.275174e-01f)); } } } } barrier(CLK_LOCAL_MEM_FENCE); for (int li_1_2 = 0; li_1_2 < 2; ++li_1_2) { for (int lj_1_2 = 0; lj_1_2 < 2; ++lj_1_2) { ((__local float*)K_smem)[((((((convert_int(get_local_id(1))) * 128) + (((convert_int(get_local_id(0))) >> 3) * 32)) + (li_1_2 * 16)) + (((convert_int(get_local_id(0))) & 7) * 2)) + lj_1_2)] = S_local[((li_1_2 * 2) + lj_1_2)]; } } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { m_prev[0] = m_smem[(convert_int(get_local_id(0)))]; m_new[0] = m_smem[(convert_int(get_local_id(0)))]; for (int j = 0; j < 16; ++j) { int cse_var_5 = ((iterator * 16) + j); bool condval_5; if ((0 < causal)) { condval_5 = (cse_var_5 <= (((((((convert_int(get_local_id(1))) * 32) + kv_chunk_len[0]) + q_indptr[(b_idx_1 + q_indptr_elem_offset)]) + LH_start) + (convert_int(get_local_id(0)))) - q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)])); } else { condval_5 = (cse_var_5 < kv_chunk_len[0]); } if (condval_5) { m_new[0] = max((float)m_new[0], (float)((__local float*)K_smem)[((((convert_int(get_local_id(1))) * 512) + ((convert_int(get_local_id(0))) * 16)) + j)]); } } d_new[0] = (d_smem[(convert_int(get_local_id(0)))] * exp2((m_prev[0] - m_new[0]))); } for (int j_1 = 0; j_1 < 16; ++j_1) { barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { int cse_var_6 = ((iterator * 16) + j_1); bool condval_6; if ((0 < causal)) { condval_6 = (cse_var_6 <= (((((((convert_int(get_local_id(1))) * 32) + kv_chunk_len[0]) + q_indptr[(b_idx_1 + q_indptr_elem_offset)]) + LH_start) + (convert_int(get_local_id(0)))) - q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)])); } else { condval_6 = (cse_var_6 < kv_chunk_len[0]); } if (condval_6) { ((__local float*)K_smem)[((((convert_int(get_local_id(1))) * 512) + ((convert_int(get_local_id(0))) * 16)) + j_1)] = exp2((((__local float*)K_smem)[((((convert_int(get_local_id(1))) * 512) + ((convert_int(get_local_id(0))) * 16)) + j_1)] - m_new[0])); } else { ((__local float*)K_smem)[(((convert_int(get_local_id(0))) * 16) + j_1)] = exp2((-5.000000e+04f - m_new[0])); } } } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { for (int j_2 = 0; j_2 < 16; ++j_2) { d_new[0] = (d_new[0] + ((__local float*)K_smem)[(((convert_int(get_local_id(0))) * 16) + j_2)]); } m_smem[(convert_int(get_local_id(0)))] = m_new[0]; d_smem[(convert_int(get_local_id(0)))] = d_new[0]; m_prev_smem[(convert_int(get_local_id(0)))] = m_prev[0]; } barrier(CLK_LOCAL_MEM_FENCE); for (int li_1_init_1 = 0; li_1_init_1 < 4; ++li_1_init_1) { for (int lj_1_init_1 = 0; lj_1_init_1 < 8; ++lj_1_init_1) { O_local[((li_1_init_1 * 8) + lj_1_init_1)] = (O_local[((li_1_init_1 * 8) + lj_1_init_1)] * exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_init_1)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_init_1)]))); } } for (int lk_0_1 = 0; lk_0_1 < 2; ++lk_0_1) { for (int lk_1_1 = 0; lk_1_1 < 8; ++lk_1_1) { for (int li_1_3 = 0; li_1_3 < 4; ++li_1_3) { for (int lj_1_3 = 0; lj_1_3 < 8; ++lj_1_3) { O_local[((li_1_3 * 8) + lj_1_3)] = (O_local[((li_1_3 * 8) + lj_1_3)] + (((__local float*)K_smem)[((((((convert_int(get_local_id(1))) * 128) + (((convert_int(get_local_id(0))) >> 4) * 64)) + (li_1_3 * 16)) + (lk_0_1 * 8)) + lk_1_1)] * (convert_float(V_smem[((((lk_0_1 * 1024) + (lk_1_1 * 128)) + (((convert_int(get_local_id(0))) & 15) * 8)) + lj_1_3)])))); } } } } } for (int li_1_4 = 0; li_1_4 < 4; ++li_1_4) { for (int lj_1_4 = 0; lj_1_4 < 8; ++lj_1_4) { int cur_L = ((((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + q_indptr[(b_idx_1 + q_indptr_elem_offset)]) + LH_start) + li_1_4); if (cur_L < q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)]) { output[((((cur_L * 2048) + ((convert_int(get_group_id(1))) * 128)) + (((convert_int(get_local_id(0))) & 15) * 8)) + lj_1_4)] = (convert_half((O_local[((li_1_4 * 8) + lj_1_4)] / d_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_4)]))); } } } if ((convert_int(get_local_id(1))) < 1) { int cur_L_1 = (((((convert_int(get_local_id(1))) * 32) + q_indptr[(b_idx_1 + q_indptr_elem_offset)]) + LH_start) + (convert_int(get_local_id(0)))); if (cur_L_1 < q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)]) { lse[((cur_L_1 * 16) + (convert_int(get_group_id(1))))] = (m_smem[(convert_int(get_local_id(0)))] + log2(d_smem[(convert_int(get_local_id(0)))])); } } tile_id[0] = (tile_id[0] + 16); } } } // Function: gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124) { T_reshape[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] = lv124[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]; } // Function: gpt_neox_q4f16_1_layer_norm_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size) { float in_thread_input_embeds_red_temp_v0_shared[1]; float in_thread_input_embeds_red_temp_v1_shared[1]; __local float red_buf0[256]; __local float red_buf1[256]; __local float input_embeds_red_temp_v0_shared[1]; __local float input_embeds_red_temp_v1_shared[1]; in_thread_input_embeds_red_temp_v0_shared[0] = 0.000000e+00f; in_thread_input_embeds_red_temp_v1_shared[0] = 0.000000e+00f; float v_input_embeds_red_temp_v0 = (in_thread_input_embeds_red_temp_v0_shared[0] + (convert_float(input_embeds[(((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0))))]))); float v_input_embeds_red_temp_v1 = (in_thread_input_embeds_red_temp_v1_shared[0] + ((convert_float(input_embeds[(((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0))))])) * (convert_float(input_embeds[(((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0))))])))); in_thread_input_embeds_red_temp_v0_shared[0] = v_input_embeds_red_temp_v0; in_thread_input_embeds_red_temp_v1_shared[0] = v_input_embeds_red_temp_v1; float v_input_embeds_red_temp_v0_1 = (in_thread_input_embeds_red_temp_v0_shared[0] + (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 256)]))); float v_input_embeds_red_temp_v1_1 = (in_thread_input_embeds_red_temp_v1_shared[0] + ((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 256)])) * (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 256)])))); in_thread_input_embeds_red_temp_v0_shared[0] = v_input_embeds_red_temp_v0_1; in_thread_input_embeds_red_temp_v1_shared[0] = v_input_embeds_red_temp_v1_1; float v_input_embeds_red_temp_v0_2 = (in_thread_input_embeds_red_temp_v0_shared[0] + (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 512)]))); float v_input_embeds_red_temp_v1_2 = (in_thread_input_embeds_red_temp_v1_shared[0] + ((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 512)])) * (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 512)])))); in_thread_input_embeds_red_temp_v0_shared[0] = v_input_embeds_red_temp_v0_2; in_thread_input_embeds_red_temp_v1_shared[0] = v_input_embeds_red_temp_v1_2; float v_input_embeds_red_temp_v0_3 = (in_thread_input_embeds_red_temp_v0_shared[0] + (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 768)]))); float v_input_embeds_red_temp_v1_3 = (in_thread_input_embeds_red_temp_v1_shared[0] + ((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 768)])) * (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 768)])))); in_thread_input_embeds_red_temp_v0_shared[0] = v_input_embeds_red_temp_v0_3; in_thread_input_embeds_red_temp_v1_shared[0] = v_input_embeds_red_temp_v1_3; float v_input_embeds_red_temp_v0_4 = (in_thread_input_embeds_red_temp_v0_shared[0] + (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1024)]))); float v_input_embeds_red_temp_v1_4 = (in_thread_input_embeds_red_temp_v1_shared[0] + ((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1024)])) * (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1024)])))); in_thread_input_embeds_red_temp_v0_shared[0] = v_input_embeds_red_temp_v0_4; in_thread_input_embeds_red_temp_v1_shared[0] = v_input_embeds_red_temp_v1_4; float v_input_embeds_red_temp_v0_5 = (in_thread_input_embeds_red_temp_v0_shared[0] + (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1280)]))); float v_input_embeds_red_temp_v1_5 = (in_thread_input_embeds_red_temp_v1_shared[0] + ((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1280)])) * (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1280)])))); in_thread_input_embeds_red_temp_v0_shared[0] = v_input_embeds_red_temp_v0_5; in_thread_input_embeds_red_temp_v1_shared[0] = v_input_embeds_red_temp_v1_5; float v_input_embeds_red_temp_v0_6 = (in_thread_input_embeds_red_temp_v0_shared[0] + (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1536)]))); float v_input_embeds_red_temp_v1_6 = (in_thread_input_embeds_red_temp_v1_shared[0] + ((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1536)])) * (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1536)])))); in_thread_input_embeds_red_temp_v0_shared[0] = v_input_embeds_red_temp_v0_6; in_thread_input_embeds_red_temp_v1_shared[0] = v_input_embeds_red_temp_v1_6; float v_input_embeds_red_temp_v0_7 = (in_thread_input_embeds_red_temp_v0_shared[0] + (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1792)]))); float v_input_embeds_red_temp_v1_7 = (in_thread_input_embeds_red_temp_v1_shared[0] + ((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1792)])) * (convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1792)])))); in_thread_input_embeds_red_temp_v0_shared[0] = v_input_embeds_red_temp_v0_7; in_thread_input_embeds_red_temp_v1_shared[0] = v_input_embeds_red_temp_v1_7; barrier(CLK_LOCAL_MEM_FENCE); ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = in_thread_input_embeds_red_temp_v0_shared[0]; ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = in_thread_input_embeds_red_temp_v1_shared[0]; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 128) { ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 128)]); ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 128)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 64) { ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 64)]); ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 64)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 32) { ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 32)]); ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 32)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 16) { ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 16)]); ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 16)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 8) { ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 8)]); ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 8)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 4) { ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 4)]); ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 4)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 2) { ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 2)]); ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 2)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 1) { ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 1)]); ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 1)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) == 0) { input_embeds_red_temp_v0_shared[0] = ((volatile __local float*)red_buf0)[0]; input_embeds_red_temp_v1_shared[0] = ((volatile __local float*)red_buf1)[0]; } barrier(CLK_LOCAL_MEM_FENCE); T_layer_norm[(((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0))))] = (((convert_half((((convert_float(input_embeds[(((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0))))])) - (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embeds_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embeds_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight4[(convert_int(get_local_id(0)))]) + gpt_neox_layers_0_input_layernorm_bias4[(convert_int(get_local_id(0)))]); T_layer_norm[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 256)] = (((convert_half((((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 256)])) - (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embeds_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embeds_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight4[((convert_int(get_local_id(0))) + 256)]) + gpt_neox_layers_0_input_layernorm_bias4[((convert_int(get_local_id(0))) + 256)]); T_layer_norm[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 512)] = (((convert_half((((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 512)])) - (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embeds_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embeds_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight4[((convert_int(get_local_id(0))) + 512)]) + gpt_neox_layers_0_input_layernorm_bias4[((convert_int(get_local_id(0))) + 512)]); T_layer_norm[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 768)] = (((convert_half((((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 768)])) - (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embeds_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embeds_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight4[((convert_int(get_local_id(0))) + 768)]) + gpt_neox_layers_0_input_layernorm_bias4[((convert_int(get_local_id(0))) + 768)]); T_layer_norm[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1024)] = (((convert_half((((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1024)])) - (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embeds_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embeds_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight4[((convert_int(get_local_id(0))) + 1024)]) + gpt_neox_layers_0_input_layernorm_bias4[((convert_int(get_local_id(0))) + 1024)]); T_layer_norm[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1280)] = (((convert_half((((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1280)])) - (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embeds_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embeds_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight4[((convert_int(get_local_id(0))) + 1280)]) + gpt_neox_layers_0_input_layernorm_bias4[((convert_int(get_local_id(0))) + 1280)]); T_layer_norm[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1536)] = (((convert_half((((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1536)])) - (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embeds_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embeds_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight4[((convert_int(get_local_id(0))) + 1536)]) + gpt_neox_layers_0_input_layernorm_bias4[((convert_int(get_local_id(0))) + 1536)]); T_layer_norm[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1792)] = (((convert_half((((convert_float(input_embeds[((((convert_int(get_group_id(0))) * 2048) + (convert_int(get_local_id(0)))) + 1792)])) - (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embeds_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embeds_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embeds_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight4[((convert_int(get_local_id(0))) + 1792)]) + gpt_neox_layers_0_input_layernorm_bias4[((convert_int(get_local_id(0))) + 1792)]); } // Function: gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294) { float NT_matmul_rf_local[8]; uint gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2_local[1]; float NT_matmul_rf_local_1[1]; __local float red_buf0[64]; vstore8(((float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)), 0, NT_matmul_rf_local + 0); for (int ax1_0_fused_ax1_1_fused_0 = 0; ax1_0_fused_ax1_1_fused_0 < 32; ++ax1_0_fused_ax1_1_fused_0) { gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2_local[0] = gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2[(((((convert_int(get_group_id(0))) * 2048) + ((convert_int(get_local_id(0))) * 1024)) + (ax1_0_fused_ax1_1_fused_0 * 32)) + (convert_int(get_local_id(1))))]; vstore8((vload8(0, NT_matmul_rf_local + 0) + ((convert_float8(vload8(0, lv294 + ((ax1_0_fused_ax1_1_fused_0 * 256) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8((((convert_half8(((((uint8)(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2_local[0], gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2_local[0], gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2_local[0], gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2_local[0], gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2_local[0], gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2_local[0], gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2_local[0], gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2_local[0])) >> ((uint8)(((uint)0)+((uint)4*0), ((uint)0)+((uint)4*1), ((uint)0)+((uint)4*2), ((uint)0)+((uint)4*3), ((uint)0)+((uint)4*4), ((uint)0)+((uint)4*5), ((uint)0)+((uint)4*6), ((uint)0)+((uint)4*7)))) & ((uint8)((uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15))))) - ((half8)((half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f))) * ((half8)(gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2[(((((convert_int(get_group_id(0))) * 512) + ((convert_int(get_local_id(0))) * 256)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2[(((((convert_int(get_group_id(0))) * 512) + ((convert_int(get_local_id(0))) * 256)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2[(((((convert_int(get_group_id(0))) * 512) + ((convert_int(get_local_id(0))) * 256)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2[(((((convert_int(get_group_id(0))) * 512) + ((convert_int(get_local_id(0))) * 256)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2[(((((convert_int(get_group_id(0))) * 512) + ((convert_int(get_local_id(0))) * 256)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2[(((((convert_int(get_group_id(0))) * 512) + ((convert_int(get_local_id(0))) * 256)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2[(((((convert_int(get_group_id(0))) * 512) + ((convert_int(get_local_id(0))) * 256)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2[(((((convert_int(get_group_id(0))) * 512) + ((convert_int(get_local_id(0))) * 256)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]))))))), 0, NT_matmul_rf_local + 0); } for (int ax2_fused_2_0 = 0; ax2_fused_2_0 < 1; ++ax2_fused_2_0) { NT_matmul_rf_local_1[0] = 0.000000e+00f; NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[0]); NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[1]); NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[2]); NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[3]); NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[4]); NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[5]); NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[6]); NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[7]); } barrier(CLK_LOCAL_MEM_FENCE); ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_rf_local_1[0]; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 16) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 8) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 4) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 2) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) == 0) { compute[(((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0))))] = (convert_half((((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) * 32)] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias2[(((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0))))]))); } } // Function: gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode) { int tile_id[1]; int batch_idx[1]; int batch_rows[1]; int batch_tiles[1]; int kv_chunk_len[1]; __local float m_smem[32]; __local float d_smem[32]; float8 O_local[4]; __local half Q_smem[4096]; __local half K_smem[4096]; __local half8 V_smem[512]; float4 S_local[2]; __local float S_smem[1024]; float m_prev[1]; float m_new[1]; float d_new[1]; __local float m_prev_smem[32]; tile_id[0] = (convert_int(get_group_id(0))); batch_idx[0] = 0; batch_rows[0] = (q_indptr[(q_indptr_elem_offset + 1)] - q_indptr[q_indptr_elem_offset]); batch_tiles[0] = ((batch_rows[0] + 31) >> 5); while (1) { if (!(((batch_idx[0] < batch_size)))) { break; } while (1) { if (!((((bool)(batch_tiles[0] <= tile_id[0])) && ((bool)(batch_idx[0] < batch_size))))) { break; } tile_id[0] = (tile_id[0] - batch_tiles[0]); batch_idx[0] = (batch_idx[0] + 1); if (batch_idx[0] < batch_size) { int b_idx = batch_idx[0]; batch_rows[0] = (q_indptr[((b_idx + q_indptr_elem_offset) + 1)] - q_indptr[(b_idx + q_indptr_elem_offset)]); batch_tiles[0] = ((batch_rows[0] + 31) >> 5); } } if ((batch_idx[0] < batch_size)) { int b_idx_1 = batch_idx[0]; int q_indptr_val = q_indptr[(b_idx_1 + q_indptr_elem_offset)]; int LH_start = (tile_id[0] * 32); kv_chunk_len[0] = (kv_indptr[((b_idx_1 + kv_indptr_elem_offset) + 1)] - kv_indptr[(b_idx_1 + kv_indptr_elem_offset)]); barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { m_smem[(convert_int(get_local_id(0)))] = -5.000000e+04f; d_smem[(convert_int(get_local_id(0)))] = 1.000000e+00f; } for (int li_1 = 0; li_1 < 4; ++li_1) { O_local[li_1] = ((float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)); } barrier(CLK_LOCAL_MEM_FENCE); for (int li_0 = 0; li_0 < 2; ++li_0) { for (int lj_0_0 = 0; lj_0_0 < 2; ++lj_0_0) { if ((((((li_0 * 16) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + q_indptr_val) + LH_start) < q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)]) { half8 condval; if ((rotary_mode == 1)) { float8 freq = (((float8)(((convert_float(q_rope_position[((((((li_0 * 16) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[((((((li_0 * 16) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[((((((li_0 * 16) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[((((((li_0 * 16) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[((((((li_0 * 16) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[((((((li_0 * 16) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[((((((li_0 * 16) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[((((((li_0 * 16) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale))) / pow(((float8)(rope_theta, rope_theta, rope_theta, rope_theta, rope_theta, rope_theta, rope_theta, rope_theta)), ((convert_float8(((int8)(((((convert_int(get_local_id(0))) & 7) * 16))+(2*0), ((((convert_int(get_local_id(0))) & 7) * 16))+(2*1), ((((convert_int(get_local_id(0))) & 7) * 16))+(2*2), ((((convert_int(get_local_id(0))) & 7) * 16))+(2*3), ((((convert_int(get_local_id(0))) & 7) * 16))+(2*4), ((((convert_int(get_local_id(0))) & 7) * 16))+(2*5), ((((convert_int(get_local_id(0))) & 7) * 16))+(2*6), ((((convert_int(get_local_id(0))) & 7) * 16))+(2*7))))) * ((float8)(7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f))))); half8 condval_1; if ((lj_0_0 < 1)) { condval_1 = (vload8(0, q + (((((((((li_0 * 32768) + ((convert_int(get_local_id(1))) * 8192)) + (((convert_int(get_local_id(0))) >> 3) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + (lj_0_0 * 64)) + (((convert_int(get_local_id(0))) & 7) * 8)) + 64)) * ((half8)((half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f))); } else { condval_1 = vload8(0, q + (((((((((li_0 * 32768) + ((convert_int(get_local_id(1))) * 8192)) + (((convert_int(get_local_id(0))) >> 3) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + (lj_0_0 * 64)) + (((convert_int(get_local_id(0))) & 7) * 8)) - 64)); } condval = (convert_half8(((cos(freq) * (convert_float8(vload8(0, q + ((((((((li_0 * 32768) + ((convert_int(get_local_id(1))) * 8192)) + (((convert_int(get_local_id(0))) >> 3) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + (lj_0_0 * 64)) + (((convert_int(get_local_id(0))) & 7) * 8)))))) + (sin(freq) * (convert_float8(condval_1)))))); } else { condval = vload8(0, q + ((((((((li_0 * 32768) + ((convert_int(get_local_id(1))) * 8192)) + (((convert_int(get_local_id(0))) >> 3) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + (lj_0_0 * 64)) + (((convert_int(get_local_id(0))) & 7) * 8))); } vstore8(condval, 0, Q_smem + (((((li_0 * 2048) + ((convert_int(get_local_id(1))) * 512)) + (((convert_int(get_local_id(0))) >> 3) * 128)) + (lj_0_0 * 64)) + (((convert_int(get_local_id(0))) & 7) * 8))); } else { vstore8(((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, Q_smem + (((((li_0 * 2048) + ((convert_int(get_local_id(1))) * 512)) + (((convert_int(get_local_id(0))) >> 3) * 128)) + (lj_0_0 * 64)) + (((convert_int(get_local_id(0))) & 7) * 8))); } } } barrier(CLK_LOCAL_MEM_FENCE); for (int iterator = 0; iterator < ((kv_chunk_len[0] + 31) >> 5); ++iterator) { int L_kv_base = kv_indptr[(b_idx_1 + kv_indptr_elem_offset)]; for (int lz_0 = 0; lz_0 < 2; ++lz_0) { for (int ly_0_0 = 0; ly_0_0 < 2; ++ly_0_0) { if (((((iterator * 32) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) < kv_chunk_len[0]) { int8 v_ = (int8)(((((((ly_0_0 * 2048) + (((convert_int(get_local_id(0))) & 7) * 256)) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)))+(32*0), ((((((ly_0_0 * 2048) + (((convert_int(get_local_id(0))) & 7) * 256)) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)))+(32*1), ((((((ly_0_0 * 2048) + (((convert_int(get_local_id(0))) & 7) * 256)) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)))+(32*2), ((((((ly_0_0 * 2048) + (((convert_int(get_local_id(0))) & 7) * 256)) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)))+(32*3), ((((((ly_0_0 * 2048) + (((convert_int(get_local_id(0))) & 7) * 256)) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)))+(32*4), ((((((ly_0_0 * 2048) + (((convert_int(get_local_id(0))) & 7) * 256)) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)))+(32*5), ((((((ly_0_0 * 2048) + (((convert_int(get_local_id(0))) & 7) * 256)) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)))+(32*6), ((((((ly_0_0 * 2048) + (((convert_int(get_local_id(0))) & 7) * 256)) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)))+(32*7)); half8 condval_2; if ((rotary_mode == 1)) { float8 freq_1 = (((float8)(((convert_float((((((iterator * 32) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + k_rope_pos_offset[(b_idx_1 + k_rope_pos_offset_elem_offset)]))) * rope_scale), ((convert_float((((((iterator * 32) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + k_rope_pos_offset[(b_idx_1 + k_rope_pos_offset_elem_offset)]))) * rope_scale), ((convert_float((((((iterator * 32) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + k_rope_pos_offset[(b_idx_1 + k_rope_pos_offset_elem_offset)]))) * rope_scale), ((convert_float((((((iterator * 32) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + k_rope_pos_offset[(b_idx_1 + k_rope_pos_offset_elem_offset)]))) * rope_scale), ((convert_float((((((iterator * 32) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + k_rope_pos_offset[(b_idx_1 + k_rope_pos_offset_elem_offset)]))) * rope_scale), ((convert_float((((((iterator * 32) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + k_rope_pos_offset[(b_idx_1 + k_rope_pos_offset_elem_offset)]))) * rope_scale), ((convert_float((((((iterator * 32) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + k_rope_pos_offset[(b_idx_1 + k_rope_pos_offset_elem_offset)]))) * rope_scale), ((convert_float((((((iterator * 32) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) + k_rope_pos_offset[(b_idx_1 + k_rope_pos_offset_elem_offset)]))) * rope_scale))) / pow(((float8)(rope_theta, rope_theta, rope_theta, rope_theta, rope_theta, rope_theta, rope_theta, rope_theta)), ((convert_float8(((int8)(((((convert_int(get_local_id(0))) & 7) * 16))+(2*0), ((((convert_int(get_local_id(0))) & 7) * 16))+(2*1), ((((convert_int(get_local_id(0))) & 7) * 16))+(2*2), ((((convert_int(get_local_id(0))) & 7) * 16))+(2*3), ((((convert_int(get_local_id(0))) & 7) * 16))+(2*4), ((((convert_int(get_local_id(0))) & 7) * 16))+(2*5), ((((convert_int(get_local_id(0))) & 7) * 16))+(2*6), ((((convert_int(get_local_id(0))) & 7) * 16))+(2*7))))) * ((float8)(7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f))))); half8 condval_3; if ((ly_0_0 < 1)) { condval_3 = (vload8(0, k + (((((((((iterator * 65536) + (lz_0 * 32768)) + ((convert_int(get_local_id(1))) * 8192)) + (((convert_int(get_local_id(0))) >> 3) * 2048)) + (L_kv_base * 2048)) + ((convert_int(get_group_id(1))) * 128)) + (ly_0_0 * 64)) + (((convert_int(get_local_id(0))) & 7) * 8)) + 64)) * ((half8)((half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f))); } else { condval_3 = vload8(0, k + (((((((((iterator * 65536) + (lz_0 * 32768)) + ((convert_int(get_local_id(1))) * 8192)) + (((convert_int(get_local_id(0))) >> 3) * 2048)) + (L_kv_base * 2048)) + ((convert_int(get_group_id(1))) * 128)) + (ly_0_0 * 64)) + (((convert_int(get_local_id(0))) & 7) * 8)) - 64)); } condval_2 = (convert_half8(((cos(freq_1) * (convert_float8(vload8(0, k + ((((((((iterator * 65536) + (lz_0 * 32768)) + ((convert_int(get_local_id(1))) * 8192)) + (((convert_int(get_local_id(0))) >> 3) * 2048)) + (L_kv_base * 2048)) + ((convert_int(get_group_id(1))) * 128)) + (ly_0_0 * 64)) + (((convert_int(get_local_id(0))) & 7) * 8)))))) + (sin(freq_1) * (convert_float8(condval_3)))))); } else { condval_2 = vload8(0, k + ((((((((iterator * 65536) + (lz_0 * 32768)) + ((convert_int(get_local_id(1))) * 8192)) + (((convert_int(get_local_id(0))) >> 3) * 2048)) + (L_kv_base * 2048)) + ((convert_int(get_group_id(1))) * 128)) + (ly_0_0 * 64)) + (((convert_int(get_local_id(0))) & 7) * 8))); } K_smem[v_.s0] = condval_2.s0; K_smem[v_.s1] = condval_2.s1; K_smem[v_.s2] = condval_2.s2; K_smem[v_.s3] = condval_2.s3; K_smem[v_.s4] = condval_2.s4; K_smem[v_.s5] = condval_2.s5; K_smem[v_.s6] = condval_2.s6; K_smem[v_.s7] = condval_2.s7; } else { int8 v__1 = (int8)(((((((ly_0_0 * 2048) + (((convert_int(get_local_id(0))) & 7) * 256)) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)))+(32*0), ((((((ly_0_0 * 2048) + (((convert_int(get_local_id(0))) & 7) * 256)) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)))+(32*1), ((((((ly_0_0 * 2048) + (((convert_int(get_local_id(0))) & 7) * 256)) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)))+(32*2), ((((((ly_0_0 * 2048) + (((convert_int(get_local_id(0))) & 7) * 256)) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)))+(32*3), ((((((ly_0_0 * 2048) + (((convert_int(get_local_id(0))) & 7) * 256)) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)))+(32*4), ((((((ly_0_0 * 2048) + (((convert_int(get_local_id(0))) & 7) * 256)) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)))+(32*5), ((((((ly_0_0 * 2048) + (((convert_int(get_local_id(0))) & 7) * 256)) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)))+(32*6), ((((((ly_0_0 * 2048) + (((convert_int(get_local_id(0))) & 7) * 256)) + (lz_0 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)))+(32*7)); half8 v__2 = (half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f); K_smem[v__1.s0] = v__2.s0; K_smem[v__1.s1] = v__2.s1; K_smem[v__1.s2] = v__2.s2; K_smem[v__1.s3] = v__2.s3; K_smem[v__1.s4] = v__2.s4; K_smem[v__1.s5] = v__2.s5; K_smem[v__1.s6] = v__2.s6; K_smem[v__1.s7] = v__2.s7; } } } barrier(CLK_LOCAL_MEM_FENCE); for (int lz_0_1 = 0; lz_0_1 < 2; ++lz_0_1) { for (int ly_0_0_1 = 0; ly_0_0_1 < 2; ++ly_0_0_1) { if (((((iterator * 32) + (lz_0_1 * 16)) + ((convert_int(get_local_id(1))) * 4)) + ((convert_int(get_local_id(0))) >> 3)) < kv_chunk_len[0]) { V_smem[(((((lz_0_1 * 256) + ((convert_int(get_local_id(1))) * 64)) + (((convert_int(get_local_id(0))) >> 3) * 16)) + (ly_0_0_1 * 8)) + ((convert_int(get_local_id(0))) & 7))] = vload8(0, v + ((((((((iterator * 65536) + (lz_0_1 * 32768)) + ((convert_int(get_local_id(1))) * 8192)) + (((convert_int(get_local_id(0))) >> 3) * 2048)) + (L_kv_base * 2048)) + ((convert_int(get_group_id(1))) * 128)) + (ly_0_0_1 * 64)) + (((convert_int(get_local_id(0))) & 7) * 8))); } else { V_smem[(((((lz_0_1 * 256) + ((convert_int(get_local_id(1))) * 64)) + (((convert_int(get_local_id(0))) >> 3) * 16)) + (ly_0_0_1 * 8)) + ((convert_int(get_local_id(0))) & 7))] = ((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)); } } } barrier(CLK_LOCAL_MEM_FENCE); S_local[0] = ((float4)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)); S_local[1] = ((float4)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)); for (int lk_0 = 0; lk_0 < 8; ++lk_0) { for (int lk_1 = 0; lk_1 < 16; ++lk_1) { S_local[0] = (S_local[0] + (((((float4)((convert_float(Q_smem[(((((convert_int(get_local_id(1))) * 1024) + (((convert_int(get_local_id(0))) >> 3) * 256)) + (lk_0 * 16)) + lk_1)])), (convert_float(Q_smem[(((((convert_int(get_local_id(1))) * 1024) + (((convert_int(get_local_id(0))) >> 3) * 256)) + (lk_0 * 16)) + lk_1)])), (convert_float(Q_smem[(((((convert_int(get_local_id(1))) * 1024) + (((convert_int(get_local_id(0))) >> 3) * 256)) + (lk_0 * 16)) + lk_1)])), (convert_float(Q_smem[(((((convert_int(get_local_id(1))) * 1024) + (((convert_int(get_local_id(0))) >> 3) * 256)) + (lk_0 * 16)) + lk_1)])))) * (convert_float4(vload4(0, K_smem + (((lk_0 * 512) + (lk_1 * 32)) + (((convert_int(get_local_id(0))) & 7) * 4)))))) * ((float4)(attn_score_scaling_factor, attn_score_scaling_factor, attn_score_scaling_factor, attn_score_scaling_factor))) * ((float4)(1.275174e-01f, 1.275174e-01f, 1.275174e-01f, 1.275174e-01f)))); } for (int lk_1_1 = 0; lk_1_1 < 16; ++lk_1_1) { S_local[1] = (S_local[1] + (((((float4)((convert_float(Q_smem[((((((convert_int(get_local_id(1))) * 1024) + (((convert_int(get_local_id(0))) >> 3) * 256)) + (lk_0 * 16)) + lk_1_1) + 128)])), (convert_float(Q_smem[((((((convert_int(get_local_id(1))) * 1024) + (((convert_int(get_local_id(0))) >> 3) * 256)) + (lk_0 * 16)) + lk_1_1) + 128)])), (convert_float(Q_smem[((((((convert_int(get_local_id(1))) * 1024) + (((convert_int(get_local_id(0))) >> 3) * 256)) + (lk_0 * 16)) + lk_1_1) + 128)])), (convert_float(Q_smem[((((((convert_int(get_local_id(1))) * 1024) + (((convert_int(get_local_id(0))) >> 3) * 256)) + (lk_0 * 16)) + lk_1_1) + 128)])))) * (convert_float4(vload4(0, K_smem + (((lk_0 * 512) + (lk_1_1 * 32)) + (((convert_int(get_local_id(0))) & 7) * 4)))))) * ((float4)(attn_score_scaling_factor, attn_score_scaling_factor, attn_score_scaling_factor, attn_score_scaling_factor))) * ((float4)(1.275174e-01f, 1.275174e-01f, 1.275174e-01f, 1.275174e-01f)))); } } barrier(CLK_LOCAL_MEM_FENCE); for (int li_1_1 = 0; li_1_1 < 2; ++li_1_1) { vstore4(S_local[li_1_1], 0, S_smem + (((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 3) * 64)) + (li_1_1 * 32)) + (((convert_int(get_local_id(0))) & 7) * 4))); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { m_prev[0] = m_smem[(convert_int(get_local_id(0)))]; m_new[0] = m_smem[(convert_int(get_local_id(0)))]; for (int j = 0; j < 32; ++j) { int cse_var_3 = ((iterator * 32) + j); bool condval_4; if ((0 < causal)) { condval_4 = (cse_var_3 <= (((((((convert_int(get_local_id(1))) * 32) + kv_chunk_len[0]) + q_indptr[(b_idx_1 + q_indptr_elem_offset)]) + LH_start) + (convert_int(get_local_id(0)))) - q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)])); } else { condval_4 = (cse_var_3 < kv_chunk_len[0]); } if (condval_4) { m_new[0] = max((float)m_new[0], (float)S_smem[((((convert_int(get_local_id(1))) * 1024) + ((convert_int(get_local_id(0))) * 32)) + j)]); } } d_new[0] = (d_smem[(convert_int(get_local_id(0)))] * exp2((m_prev[0] - m_new[0]))); } for (int j_1 = 0; j_1 < 32; ++j_1) { barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { int cse_var_4 = ((iterator * 32) + j_1); bool condval_5; if ((0 < causal)) { condval_5 = (cse_var_4 <= (((((((convert_int(get_local_id(1))) * 32) + kv_chunk_len[0]) + q_indptr[(b_idx_1 + q_indptr_elem_offset)]) + LH_start) + (convert_int(get_local_id(0)))) - q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)])); } else { condval_5 = (cse_var_4 < kv_chunk_len[0]); } if (condval_5) { S_smem[((((convert_int(get_local_id(1))) * 1024) + ((convert_int(get_local_id(0))) * 32)) + j_1)] = exp2((S_smem[((((convert_int(get_local_id(1))) * 1024) + ((convert_int(get_local_id(0))) * 32)) + j_1)] - m_new[0])); } else { S_smem[(((convert_int(get_local_id(0))) * 32) + j_1)] = exp2((-5.000000e+04f - m_new[0])); } } } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { for (int j_2 = 0; j_2 < 32; ++j_2) { d_new[0] = (d_new[0] + S_smem[(((convert_int(get_local_id(0))) * 32) + j_2)]); } m_smem[(convert_int(get_local_id(0)))] = m_new[0]; d_smem[(convert_int(get_local_id(0)))] = d_new[0]; m_prev_smem[(convert_int(get_local_id(0)))] = m_prev[0]; } barrier(CLK_LOCAL_MEM_FENCE); O_local[0] = (O_local[0] * ((float8)(exp2((m_prev_smem[(((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4))] - m_smem[(((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4))])), exp2((m_prev_smem[(((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4))] - m_smem[(((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4))])), exp2((m_prev_smem[(((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4))] - m_smem[(((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4))])), exp2((m_prev_smem[(((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4))] - m_smem[(((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4))])), exp2((m_prev_smem[(((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4))] - m_smem[(((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4))])), exp2((m_prev_smem[(((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4))] - m_smem[(((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4))])), exp2((m_prev_smem[(((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4))] - m_smem[(((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4))])), exp2((m_prev_smem[(((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4))] - m_smem[(((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4))]))))); O_local[1] = (O_local[1] * ((float8)(exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 1)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 1)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 1)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 1)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 1)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 1)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 1)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 1)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 1)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 1)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 1)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 1)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 1)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 1)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 1)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 1)]))))); O_local[2] = (O_local[2] * ((float8)(exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 2)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 2)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 2)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 2)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 2)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 2)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 2)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 2)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 2)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 2)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 2)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 2)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 2)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 2)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 2)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 2)]))))); O_local[3] = (O_local[3] * ((float8)(exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 3)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 3)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 3)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 3)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 3)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 3)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 3)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 3)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 3)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 3)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 3)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 3)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 3)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 3)])), exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 3)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + 3)]))))); for (int lk_0_1 = 0; lk_0_1 < 2; ++lk_0_1) { for (int lk_1_2 = 0; lk_1_2 < 16; ++lk_1_2) { O_local[0] = (O_local[0] + (((float8)(S_smem[(((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2)], S_smem[(((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2)], S_smem[(((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2)], S_smem[(((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2)], S_smem[(((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2)], S_smem[(((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2)], S_smem[(((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2)], S_smem[(((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2)])) * (convert_float8(V_smem[(((lk_0_1 * 256) + (lk_1_2 * 16)) + ((convert_int(get_local_id(0))) & 15))])))); O_local[1] = (O_local[1] + (((float8)(S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 32)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 32)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 32)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 32)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 32)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 32)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 32)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 32)])) * (convert_float8(V_smem[(((lk_0_1 * 256) + (lk_1_2 * 16)) + ((convert_int(get_local_id(0))) & 15))])))); O_local[2] = (O_local[2] + (((float8)(S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 64)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 64)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 64)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 64)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 64)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 64)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 64)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 64)])) * (convert_float8(V_smem[(((lk_0_1 * 256) + (lk_1_2 * 16)) + ((convert_int(get_local_id(0))) & 15))])))); O_local[3] = (O_local[3] + (((float8)(S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 96)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 96)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 96)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 96)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 96)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 96)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 96)], S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (lk_0_1 * 16)) + lk_1_2) + 96)])) * (convert_float8(V_smem[(((lk_0_1 * 256) + (lk_1_2 * 16)) + ((convert_int(get_local_id(0))) & 15))])))); } } } for (int li_1_2 = 0; li_1_2 < 4; ++li_1_2) { int cur_L = ((((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + q_indptr[(b_idx_1 + q_indptr_elem_offset)]) + LH_start) + li_1_2); if (cur_L < q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)]) { vstore8((convert_half8((O_local[li_1_2] / ((float8)(d_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_2)], d_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_2)], d_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_2)], d_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_2)], d_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_2)], d_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_2)], d_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_2)], d_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_2)]))))), 0, output + (((cur_L * 2048) + ((convert_int(get_group_id(1))) * 128)) + (((convert_int(get_local_id(0))) & 15) * 8))); } } if ((convert_int(get_local_id(1))) < 1) { int cur_L_1 = (((((convert_int(get_local_id(1))) * 32) + q_indptr[(b_idx_1 + q_indptr_elem_offset)]) + LH_start) + (convert_int(get_local_id(0)))); if (cur_L_1 < q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)]) { lse[((cur_L_1 * 16) + (convert_int(get_group_id(1))))] = (m_smem[(convert_int(get_local_id(0)))] + log2(d_smem[(convert_int(get_local_id(0)))])); } } tile_id[0] = (tile_id[0] + 8); } } } // Function: gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2 #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size) { half8 NT_matmul_intermediate_reindex_pad_local[2]; for (int var = 0; var < 1; ++var) { NT_matmul_intermediate_reindex_pad_local[0] = ((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)); NT_matmul_intermediate_reindex_pad_local[1] = ((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)); for (int ax3_0 = 0; ax3_0 < 128; ++ax3_0) { for (int ax3_1 = 0; ax3_1 < 16; ++ax3_1) { uint cse_var_7 = (convert_uint(((ax3_1 & 7) * 4))); half condval; if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { condval = layer_norm147[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; } else { condval = (half)0.000000e+00f; } int8 v_ = (int8)(((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*0), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*1), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*2), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*3), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*4), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*5), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*6), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*7)); int8 v__1 = (int8)((((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*0), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*1), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*2), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*3), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*4), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*5), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*6), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*7)); NT_matmul_intermediate_reindex_pad_local[0] = (NT_matmul_intermediate_reindex_pad_local[0] + (((half8)(condval, condval, condval, condval, condval, condval, condval, condval)) * (((convert_half8(((((uint8)(gpt_neox_layers_0_attention_query_key_value_q_weight4[v_.s0],gpt_neox_layers_0_attention_query_key_value_q_weight4[v_.s1],gpt_neox_layers_0_attention_query_key_value_q_weight4[v_.s2],gpt_neox_layers_0_attention_query_key_value_q_weight4[v_.s3],gpt_neox_layers_0_attention_query_key_value_q_weight4[v_.s4],gpt_neox_layers_0_attention_query_key_value_q_weight4[v_.s5],gpt_neox_layers_0_attention_query_key_value_q_weight4[v_.s6],gpt_neox_layers_0_attention_query_key_value_q_weight4[v_.s7])) >> ((uint8)(cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7))) & ((uint8)((uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15))))) - ((half8)((half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f))) * ((half8)(gpt_neox_layers_0_attention_query_key_value_q_scale4[v__1.s0],gpt_neox_layers_0_attention_query_key_value_q_scale4[v__1.s1],gpt_neox_layers_0_attention_query_key_value_q_scale4[v__1.s2],gpt_neox_layers_0_attention_query_key_value_q_scale4[v__1.s3],gpt_neox_layers_0_attention_query_key_value_q_scale4[v__1.s4],gpt_neox_layers_0_attention_query_key_value_q_scale4[v__1.s5],gpt_neox_layers_0_attention_query_key_value_q_scale4[v__1.s6],gpt_neox_layers_0_attention_query_key_value_q_scale4[v__1.s7]))))); half condval_1; if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { condval_1 = layer_norm147[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; } else { condval_1 = (half)0.000000e+00f; } int8 v__2 = (int8)(((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*0), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*1), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*2), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*3), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*4), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*5), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*6), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*7)); int8 v__3 = (int8)((((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*0), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*1), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*2), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*3), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*4), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*5), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*6), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*7)); NT_matmul_intermediate_reindex_pad_local[1] = (NT_matmul_intermediate_reindex_pad_local[1] + (((half8)(condval_1, condval_1, condval_1, condval_1, condval_1, condval_1, condval_1, condval_1)) * (((convert_half8(((((uint8)(gpt_neox_layers_0_attention_query_key_value_q_weight4[v__2.s0],gpt_neox_layers_0_attention_query_key_value_q_weight4[v__2.s1],gpt_neox_layers_0_attention_query_key_value_q_weight4[v__2.s2],gpt_neox_layers_0_attention_query_key_value_q_weight4[v__2.s3],gpt_neox_layers_0_attention_query_key_value_q_weight4[v__2.s4],gpt_neox_layers_0_attention_query_key_value_q_weight4[v__2.s5],gpt_neox_layers_0_attention_query_key_value_q_weight4[v__2.s6],gpt_neox_layers_0_attention_query_key_value_q_weight4[v__2.s7])) >> ((uint8)(cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7))) & ((uint8)((uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15))))) - ((half8)((half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f))) * ((half8)(gpt_neox_layers_0_attention_query_key_value_q_scale4[v__3.s0],gpt_neox_layers_0_attention_query_key_value_q_scale4[v__3.s1],gpt_neox_layers_0_attention_query_key_value_q_scale4[v__3.s2],gpt_neox_layers_0_attention_query_key_value_q_scale4[v__3.s3],gpt_neox_layers_0_attention_query_key_value_q_scale4[v__3.s4],gpt_neox_layers_0_attention_query_key_value_q_scale4[v__3.s5],gpt_neox_layers_0_attention_query_key_value_q_scale4[v__3.s6],gpt_neox_layers_0_attention_query_key_value_q_scale4[v__3.s7]))))); } } if ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size) { vstore8((NT_matmul_intermediate_reindex_pad_local[0] + vload8(0, gpt_neox_layers_0_attention_query_key_value_bias4 + (((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)))), 0, T_add + (((((convert_int(get_group_id(1))) * 49152) + ((convert_int(get_local_id(1))) * 12288)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8))); } if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size) { vstore8((NT_matmul_intermediate_reindex_pad_local[1] + vload8(0, gpt_neox_layers_0_attention_query_key_value_bias4 + (((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)))), 0, T_add + ((((((convert_int(get_group_id(1))) * 49152) + ((convert_int(get_local_id(1))) * 12288)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 6144)); } } } // Function: gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1 #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size) { half NT_matmul_intermediate_pad_rf_local[64]; half dequantize_local[16]; half NT_matmul_intermediate_pad_rf_local_1[8]; half NT_matmul_intermediate_pad_local[8]; __local half red_buf0[64]; for (int ax0_1_init = 0; ax0_1_init < 4; ++ax0_1_init) { for (int ax1_fused_2_init = 0; ax1_fused_2_init < 2; ++ax1_fused_2_init) { int8 v_ = (int8)((((ax0_1_init * 2) + ax1_fused_2_init))+(8*0), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*1), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*2), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*3), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*4), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*5), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*6), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*7)); half8 v__1 = (half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f); NT_matmul_intermediate_pad_rf_local[v_.s0] = v__1.s0; NT_matmul_intermediate_pad_rf_local[v_.s1] = v__1.s1; NT_matmul_intermediate_pad_rf_local[v_.s2] = v__1.s2; NT_matmul_intermediate_pad_rf_local[v_.s3] = v__1.s3; NT_matmul_intermediate_pad_rf_local[v_.s4] = v__1.s4; NT_matmul_intermediate_pad_rf_local[v_.s5] = v__1.s5; NT_matmul_intermediate_pad_rf_local[v_.s6] = v__1.s6; NT_matmul_intermediate_pad_rf_local[v_.s7] = v__1.s7; } } for (int ax2_fused_0 = 0; ax2_fused_0 < 8; ++ax2_fused_0) { for (int ax0_0 = 0; ax0_0 < 2; ++ax0_0) { dequantize_local[(ax0_0 * 8)] = (((convert_half(((gpt_neox_layers_0_attention_query_key_value_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)0) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_query_key_value_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 1)] = (((convert_half(((gpt_neox_layers_0_attention_query_key_value_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)4) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_query_key_value_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 2)] = (((convert_half(((gpt_neox_layers_0_attention_query_key_value_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)8) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_query_key_value_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 3)] = (((convert_half(((gpt_neox_layers_0_attention_query_key_value_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)12) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_query_key_value_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 4)] = (((convert_half(((gpt_neox_layers_0_attention_query_key_value_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)16) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_query_key_value_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 5)] = (((convert_half(((gpt_neox_layers_0_attention_query_key_value_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)20) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_query_key_value_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 6)] = (((convert_half(((gpt_neox_layers_0_attention_query_key_value_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)24) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_query_key_value_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 7)] = (((convert_half(((gpt_neox_layers_0_attention_query_key_value_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)28) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_query_key_value_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); } if (((convert_int(get_group_id(1))) * 4) < batch_size) { int8 v__2 = (int8)((0)+(8*0), (0)+(8*1), (0)+(8*2), (0)+(8*3), (0)+(8*4), (0)+(8*5), (0)+(8*6), (0)+(8*7)); half8 v__3 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__2.s0],NT_matmul_intermediate_pad_rf_local[v__2.s1],NT_matmul_intermediate_pad_rf_local[v__2.s2],NT_matmul_intermediate_pad_rf_local[v__2.s3],NT_matmul_intermediate_pad_rf_local[v__2.s4],NT_matmul_intermediate_pad_rf_local[v__2.s5],NT_matmul_intermediate_pad_rf_local[v__2.s6],NT_matmul_intermediate_pad_rf_local[v__2.s7])) + (vload8(0, layer_norm147 + ((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, dequantize_local + 0)); NT_matmul_intermediate_pad_rf_local[v__2.s0] = v__3.s0; NT_matmul_intermediate_pad_rf_local[v__2.s1] = v__3.s1; NT_matmul_intermediate_pad_rf_local[v__2.s2] = v__3.s2; NT_matmul_intermediate_pad_rf_local[v__2.s3] = v__3.s3; NT_matmul_intermediate_pad_rf_local[v__2.s4] = v__3.s4; NT_matmul_intermediate_pad_rf_local[v__2.s5] = v__3.s5; NT_matmul_intermediate_pad_rf_local[v__2.s6] = v__3.s6; NT_matmul_intermediate_pad_rf_local[v__2.s7] = v__3.s7; int8 v__4 = (int8)((1)+(8*0), (1)+(8*1), (1)+(8*2), (1)+(8*3), (1)+(8*4), (1)+(8*5), (1)+(8*6), (1)+(8*7)); half8 v__5 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__4.s0],NT_matmul_intermediate_pad_rf_local[v__4.s1],NT_matmul_intermediate_pad_rf_local[v__4.s2],NT_matmul_intermediate_pad_rf_local[v__4.s3],NT_matmul_intermediate_pad_rf_local[v__4.s4],NT_matmul_intermediate_pad_rf_local[v__4.s5],NT_matmul_intermediate_pad_rf_local[v__4.s6],NT_matmul_intermediate_pad_rf_local[v__4.s7])) + (vload8(0, layer_norm147 + ((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, dequantize_local + 8)); NT_matmul_intermediate_pad_rf_local[v__4.s0] = v__5.s0; NT_matmul_intermediate_pad_rf_local[v__4.s1] = v__5.s1; NT_matmul_intermediate_pad_rf_local[v__4.s2] = v__5.s2; NT_matmul_intermediate_pad_rf_local[v__4.s3] = v__5.s3; NT_matmul_intermediate_pad_rf_local[v__4.s4] = v__5.s4; NT_matmul_intermediate_pad_rf_local[v__4.s5] = v__5.s5; NT_matmul_intermediate_pad_rf_local[v__4.s6] = v__5.s6; NT_matmul_intermediate_pad_rf_local[v__4.s7] = v__5.s7; } if ((((convert_int(get_group_id(1))) * 4) + 1) < batch_size) { int8 v__6 = (int8)((2)+(8*0), (2)+(8*1), (2)+(8*2), (2)+(8*3), (2)+(8*4), (2)+(8*5), (2)+(8*6), (2)+(8*7)); half8 v__7 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__6.s0],NT_matmul_intermediate_pad_rf_local[v__6.s1],NT_matmul_intermediate_pad_rf_local[v__6.s2],NT_matmul_intermediate_pad_rf_local[v__6.s3],NT_matmul_intermediate_pad_rf_local[v__6.s4],NT_matmul_intermediate_pad_rf_local[v__6.s5],NT_matmul_intermediate_pad_rf_local[v__6.s6],NT_matmul_intermediate_pad_rf_local[v__6.s7])) + (vload8(0, layer_norm147 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)) * vload8(0, dequantize_local + 0)); NT_matmul_intermediate_pad_rf_local[v__6.s0] = v__7.s0; NT_matmul_intermediate_pad_rf_local[v__6.s1] = v__7.s1; NT_matmul_intermediate_pad_rf_local[v__6.s2] = v__7.s2; NT_matmul_intermediate_pad_rf_local[v__6.s3] = v__7.s3; NT_matmul_intermediate_pad_rf_local[v__6.s4] = v__7.s4; NT_matmul_intermediate_pad_rf_local[v__6.s5] = v__7.s5; NT_matmul_intermediate_pad_rf_local[v__6.s6] = v__7.s6; NT_matmul_intermediate_pad_rf_local[v__6.s7] = v__7.s7; int8 v__8 = (int8)((3)+(8*0), (3)+(8*1), (3)+(8*2), (3)+(8*3), (3)+(8*4), (3)+(8*5), (3)+(8*6), (3)+(8*7)); half8 v__9 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__8.s0],NT_matmul_intermediate_pad_rf_local[v__8.s1],NT_matmul_intermediate_pad_rf_local[v__8.s2],NT_matmul_intermediate_pad_rf_local[v__8.s3],NT_matmul_intermediate_pad_rf_local[v__8.s4],NT_matmul_intermediate_pad_rf_local[v__8.s5],NT_matmul_intermediate_pad_rf_local[v__8.s6],NT_matmul_intermediate_pad_rf_local[v__8.s7])) + (vload8(0, layer_norm147 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)) * vload8(0, dequantize_local + 8)); NT_matmul_intermediate_pad_rf_local[v__8.s0] = v__9.s0; NT_matmul_intermediate_pad_rf_local[v__8.s1] = v__9.s1; NT_matmul_intermediate_pad_rf_local[v__8.s2] = v__9.s2; NT_matmul_intermediate_pad_rf_local[v__8.s3] = v__9.s3; NT_matmul_intermediate_pad_rf_local[v__8.s4] = v__9.s4; NT_matmul_intermediate_pad_rf_local[v__8.s5] = v__9.s5; NT_matmul_intermediate_pad_rf_local[v__8.s6] = v__9.s6; NT_matmul_intermediate_pad_rf_local[v__8.s7] = v__9.s7; } if ((((convert_int(get_group_id(1))) * 4) + 2) < batch_size) { int8 v__10 = (int8)((4)+(8*0), (4)+(8*1), (4)+(8*2), (4)+(8*3), (4)+(8*4), (4)+(8*5), (4)+(8*6), (4)+(8*7)); half8 v__11 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__10.s0],NT_matmul_intermediate_pad_rf_local[v__10.s1],NT_matmul_intermediate_pad_rf_local[v__10.s2],NT_matmul_intermediate_pad_rf_local[v__10.s3],NT_matmul_intermediate_pad_rf_local[v__10.s4],NT_matmul_intermediate_pad_rf_local[v__10.s5],NT_matmul_intermediate_pad_rf_local[v__10.s6],NT_matmul_intermediate_pad_rf_local[v__10.s7])) + (vload8(0, layer_norm147 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 4096)) * vload8(0, dequantize_local + 0)); NT_matmul_intermediate_pad_rf_local[v__10.s0] = v__11.s0; NT_matmul_intermediate_pad_rf_local[v__10.s1] = v__11.s1; NT_matmul_intermediate_pad_rf_local[v__10.s2] = v__11.s2; NT_matmul_intermediate_pad_rf_local[v__10.s3] = v__11.s3; NT_matmul_intermediate_pad_rf_local[v__10.s4] = v__11.s4; NT_matmul_intermediate_pad_rf_local[v__10.s5] = v__11.s5; NT_matmul_intermediate_pad_rf_local[v__10.s6] = v__11.s6; NT_matmul_intermediate_pad_rf_local[v__10.s7] = v__11.s7; int8 v__12 = (int8)((5)+(8*0), (5)+(8*1), (5)+(8*2), (5)+(8*3), (5)+(8*4), (5)+(8*5), (5)+(8*6), (5)+(8*7)); half8 v__13 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__12.s0],NT_matmul_intermediate_pad_rf_local[v__12.s1],NT_matmul_intermediate_pad_rf_local[v__12.s2],NT_matmul_intermediate_pad_rf_local[v__12.s3],NT_matmul_intermediate_pad_rf_local[v__12.s4],NT_matmul_intermediate_pad_rf_local[v__12.s5],NT_matmul_intermediate_pad_rf_local[v__12.s6],NT_matmul_intermediate_pad_rf_local[v__12.s7])) + (vload8(0, layer_norm147 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 4096)) * vload8(0, dequantize_local + 8)); NT_matmul_intermediate_pad_rf_local[v__12.s0] = v__13.s0; NT_matmul_intermediate_pad_rf_local[v__12.s1] = v__13.s1; NT_matmul_intermediate_pad_rf_local[v__12.s2] = v__13.s2; NT_matmul_intermediate_pad_rf_local[v__12.s3] = v__13.s3; NT_matmul_intermediate_pad_rf_local[v__12.s4] = v__13.s4; NT_matmul_intermediate_pad_rf_local[v__12.s5] = v__13.s5; NT_matmul_intermediate_pad_rf_local[v__12.s6] = v__13.s6; NT_matmul_intermediate_pad_rf_local[v__12.s7] = v__13.s7; } if ((((convert_int(get_group_id(1))) * 4) + 3) < batch_size) { int8 v__14 = (int8)((6)+(8*0), (6)+(8*1), (6)+(8*2), (6)+(8*3), (6)+(8*4), (6)+(8*5), (6)+(8*6), (6)+(8*7)); half8 v__15 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__14.s0],NT_matmul_intermediate_pad_rf_local[v__14.s1],NT_matmul_intermediate_pad_rf_local[v__14.s2],NT_matmul_intermediate_pad_rf_local[v__14.s3],NT_matmul_intermediate_pad_rf_local[v__14.s4],NT_matmul_intermediate_pad_rf_local[v__14.s5],NT_matmul_intermediate_pad_rf_local[v__14.s6],NT_matmul_intermediate_pad_rf_local[v__14.s7])) + (vload8(0, layer_norm147 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 6144)) * vload8(0, dequantize_local + 0)); NT_matmul_intermediate_pad_rf_local[v__14.s0] = v__15.s0; NT_matmul_intermediate_pad_rf_local[v__14.s1] = v__15.s1; NT_matmul_intermediate_pad_rf_local[v__14.s2] = v__15.s2; NT_matmul_intermediate_pad_rf_local[v__14.s3] = v__15.s3; NT_matmul_intermediate_pad_rf_local[v__14.s4] = v__15.s4; NT_matmul_intermediate_pad_rf_local[v__14.s5] = v__15.s5; NT_matmul_intermediate_pad_rf_local[v__14.s6] = v__15.s6; NT_matmul_intermediate_pad_rf_local[v__14.s7] = v__15.s7; int8 v__16 = (int8)((7)+(8*0), (7)+(8*1), (7)+(8*2), (7)+(8*3), (7)+(8*4), (7)+(8*5), (7)+(8*6), (7)+(8*7)); half8 v__17 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__16.s0],NT_matmul_intermediate_pad_rf_local[v__16.s1],NT_matmul_intermediate_pad_rf_local[v__16.s2],NT_matmul_intermediate_pad_rf_local[v__16.s3],NT_matmul_intermediate_pad_rf_local[v__16.s4],NT_matmul_intermediate_pad_rf_local[v__16.s5],NT_matmul_intermediate_pad_rf_local[v__16.s6],NT_matmul_intermediate_pad_rf_local[v__16.s7])) + (vload8(0, layer_norm147 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 6144)) * vload8(0, dequantize_local + 8)); NT_matmul_intermediate_pad_rf_local[v__16.s0] = v__17.s0; NT_matmul_intermediate_pad_rf_local[v__16.s1] = v__17.s1; NT_matmul_intermediate_pad_rf_local[v__16.s2] = v__17.s2; NT_matmul_intermediate_pad_rf_local[v__16.s3] = v__17.s3; NT_matmul_intermediate_pad_rf_local[v__16.s4] = v__17.s4; NT_matmul_intermediate_pad_rf_local[v__16.s5] = v__17.s5; NT_matmul_intermediate_pad_rf_local[v__16.s6] = v__17.s6; NT_matmul_intermediate_pad_rf_local[v__16.s7] = v__17.s7; } } for (int ax3_fused_2_0 = 0; ax3_fused_2_0 < 1; ++ax3_fused_2_0) { for (int ax2 = 0; ax2 < 4; ++ax2) { vstore2(((half2)((half)0.000000e+00f, (half)0.000000e+00f)), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + (ax2 * 2))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 8))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 16))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 24))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 32))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 40))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 48))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 56))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); } } for (int ax2_fused_2 = 0; ax2_fused_2 < 2; ++ax2_fused_2) { for (int ax1 = 0; ax1 < 4; ++ax1) { barrier(CLK_LOCAL_MEM_FENCE); ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_intermediate_pad_rf_local_1[((ax1 * 2) + ax2_fused_2)]; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 16) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 8) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 4) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 2) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); } barrier(CLK_LOCAL_MEM_FENCE); NT_matmul_intermediate_pad_local[((ax1 * 2) + ax2_fused_2)] = ((volatile __local half*)red_buf0)[((convert_int(get_local_id(0))) * 32)]; } } for (int ax0 = 0; ax0 < 4; ++ax0) { for (int ax1_fused_2 = 0; ax1_fused_2 < 2; ++ax1_fused_2) { if (((bool)(((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)(((bool)(((convert_int(get_group_id(1))) - ((batch_size + 3) >> 2)) < 0)) || ((bool)((convert_int(get_group_id(1))) == 0)))))) && ((bool)((((convert_int(get_group_id(1))) * 4) + ax0) < batch_size))) { T_add[((((((convert_int(get_group_id(1))) * 24576) + (ax0 * 6144)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] = (NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_attention_query_key_value_bias4[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]); } } } } // Function: gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size) { half NT_matmul_intermediate_pad_rf_local[32]; half dequantize_local[16]; half NT_matmul_intermediate_pad_rf_local_1[4]; half NT_matmul_intermediate_pad_local[4]; __local half red_buf0[64]; for (int ax0_1_init = 0; ax0_1_init < 2; ++ax0_1_init) { for (int ax1_fused_2_init = 0; ax1_fused_2_init < 2; ++ax1_fused_2_init) { int8 v_ = (int8)((((ax0_1_init * 2) + ax1_fused_2_init))+(4*0), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*1), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*2), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*3), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*4), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*5), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*6), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*7)); half8 v__1 = (half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f); NT_matmul_intermediate_pad_rf_local[v_.s0] = v__1.s0; NT_matmul_intermediate_pad_rf_local[v_.s1] = v__1.s1; NT_matmul_intermediate_pad_rf_local[v_.s2] = v__1.s2; NT_matmul_intermediate_pad_rf_local[v_.s3] = v__1.s3; NT_matmul_intermediate_pad_rf_local[v_.s4] = v__1.s4; NT_matmul_intermediate_pad_rf_local[v_.s5] = v__1.s5; NT_matmul_intermediate_pad_rf_local[v_.s6] = v__1.s6; NT_matmul_intermediate_pad_rf_local[v_.s7] = v__1.s7; } } for (int ax2_fused_0 = 0; ax2_fused_0 < 8; ++ax2_fused_0) { for (int ax0_0 = 0; ax0_0 < 2; ++ax0_0) { dequantize_local[(ax0_0 * 8)] = (((convert_half(((gpt_neox_layers_0_attention_dense_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)0) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_dense_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 1)] = (((convert_half(((gpt_neox_layers_0_attention_dense_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)4) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_dense_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 2)] = (((convert_half(((gpt_neox_layers_0_attention_dense_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)8) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_dense_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 3)] = (((convert_half(((gpt_neox_layers_0_attention_dense_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)12) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_dense_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 4)] = (((convert_half(((gpt_neox_layers_0_attention_dense_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)16) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_dense_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 5)] = (((convert_half(((gpt_neox_layers_0_attention_dense_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)20) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_dense_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 6)] = (((convert_half(((gpt_neox_layers_0_attention_dense_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)24) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_dense_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 7)] = (((convert_half(((gpt_neox_layers_0_attention_dense_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)28) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_dense_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); } if (((convert_int(get_group_id(1))) * 2) < batch_size) { int8 v__2 = (int8)((0)+(4*0), (0)+(4*1), (0)+(4*2), (0)+(4*3), (0)+(4*4), (0)+(4*5), (0)+(4*6), (0)+(4*7)); half8 v__3 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__2.s0],NT_matmul_intermediate_pad_rf_local[v__2.s1],NT_matmul_intermediate_pad_rf_local[v__2.s2],NT_matmul_intermediate_pad_rf_local[v__2.s3],NT_matmul_intermediate_pad_rf_local[v__2.s4],NT_matmul_intermediate_pad_rf_local[v__2.s5],NT_matmul_intermediate_pad_rf_local[v__2.s6],NT_matmul_intermediate_pad_rf_local[v__2.s7])) + (vload8(0, reshape291 + ((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, dequantize_local + 0)); NT_matmul_intermediate_pad_rf_local[v__2.s0] = v__3.s0; NT_matmul_intermediate_pad_rf_local[v__2.s1] = v__3.s1; NT_matmul_intermediate_pad_rf_local[v__2.s2] = v__3.s2; NT_matmul_intermediate_pad_rf_local[v__2.s3] = v__3.s3; NT_matmul_intermediate_pad_rf_local[v__2.s4] = v__3.s4; NT_matmul_intermediate_pad_rf_local[v__2.s5] = v__3.s5; NT_matmul_intermediate_pad_rf_local[v__2.s6] = v__3.s6; NT_matmul_intermediate_pad_rf_local[v__2.s7] = v__3.s7; int8 v__4 = (int8)((1)+(4*0), (1)+(4*1), (1)+(4*2), (1)+(4*3), (1)+(4*4), (1)+(4*5), (1)+(4*6), (1)+(4*7)); half8 v__5 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__4.s0],NT_matmul_intermediate_pad_rf_local[v__4.s1],NT_matmul_intermediate_pad_rf_local[v__4.s2],NT_matmul_intermediate_pad_rf_local[v__4.s3],NT_matmul_intermediate_pad_rf_local[v__4.s4],NT_matmul_intermediate_pad_rf_local[v__4.s5],NT_matmul_intermediate_pad_rf_local[v__4.s6],NT_matmul_intermediate_pad_rf_local[v__4.s7])) + (vload8(0, reshape291 + ((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, dequantize_local + 8)); NT_matmul_intermediate_pad_rf_local[v__4.s0] = v__5.s0; NT_matmul_intermediate_pad_rf_local[v__4.s1] = v__5.s1; NT_matmul_intermediate_pad_rf_local[v__4.s2] = v__5.s2; NT_matmul_intermediate_pad_rf_local[v__4.s3] = v__5.s3; NT_matmul_intermediate_pad_rf_local[v__4.s4] = v__5.s4; NT_matmul_intermediate_pad_rf_local[v__4.s5] = v__5.s5; NT_matmul_intermediate_pad_rf_local[v__4.s6] = v__5.s6; NT_matmul_intermediate_pad_rf_local[v__4.s7] = v__5.s7; } if ((((convert_int(get_group_id(1))) * 2) + 1) < batch_size) { int8 v__6 = (int8)((2)+(4*0), (2)+(4*1), (2)+(4*2), (2)+(4*3), (2)+(4*4), (2)+(4*5), (2)+(4*6), (2)+(4*7)); half8 v__7 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__6.s0],NT_matmul_intermediate_pad_rf_local[v__6.s1],NT_matmul_intermediate_pad_rf_local[v__6.s2],NT_matmul_intermediate_pad_rf_local[v__6.s3],NT_matmul_intermediate_pad_rf_local[v__6.s4],NT_matmul_intermediate_pad_rf_local[v__6.s5],NT_matmul_intermediate_pad_rf_local[v__6.s6],NT_matmul_intermediate_pad_rf_local[v__6.s7])) + (vload8(0, reshape291 + (((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)) * vload8(0, dequantize_local + 0)); NT_matmul_intermediate_pad_rf_local[v__6.s0] = v__7.s0; NT_matmul_intermediate_pad_rf_local[v__6.s1] = v__7.s1; NT_matmul_intermediate_pad_rf_local[v__6.s2] = v__7.s2; NT_matmul_intermediate_pad_rf_local[v__6.s3] = v__7.s3; NT_matmul_intermediate_pad_rf_local[v__6.s4] = v__7.s4; NT_matmul_intermediate_pad_rf_local[v__6.s5] = v__7.s5; NT_matmul_intermediate_pad_rf_local[v__6.s6] = v__7.s6; NT_matmul_intermediate_pad_rf_local[v__6.s7] = v__7.s7; int8 v__8 = (int8)((3)+(4*0), (3)+(4*1), (3)+(4*2), (3)+(4*3), (3)+(4*4), (3)+(4*5), (3)+(4*6), (3)+(4*7)); half8 v__9 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__8.s0],NT_matmul_intermediate_pad_rf_local[v__8.s1],NT_matmul_intermediate_pad_rf_local[v__8.s2],NT_matmul_intermediate_pad_rf_local[v__8.s3],NT_matmul_intermediate_pad_rf_local[v__8.s4],NT_matmul_intermediate_pad_rf_local[v__8.s5],NT_matmul_intermediate_pad_rf_local[v__8.s6],NT_matmul_intermediate_pad_rf_local[v__8.s7])) + (vload8(0, reshape291 + (((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)) * vload8(0, dequantize_local + 8)); NT_matmul_intermediate_pad_rf_local[v__8.s0] = v__9.s0; NT_matmul_intermediate_pad_rf_local[v__8.s1] = v__9.s1; NT_matmul_intermediate_pad_rf_local[v__8.s2] = v__9.s2; NT_matmul_intermediate_pad_rf_local[v__8.s3] = v__9.s3; NT_matmul_intermediate_pad_rf_local[v__8.s4] = v__9.s4; NT_matmul_intermediate_pad_rf_local[v__8.s5] = v__9.s5; NT_matmul_intermediate_pad_rf_local[v__8.s6] = v__9.s6; NT_matmul_intermediate_pad_rf_local[v__8.s7] = v__9.s7; } } for (int ax3_fused_2_0 = 0; ax3_fused_2_0 < 1; ++ax3_fused_2_0) { for (int ax2 = 0; ax2 < 2; ++ax2) { vstore2(((half2)((half)0.000000e+00f, (half)0.000000e+00f)), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + (ax2 * 2))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 4))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 8))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 12))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 16))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 20))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 24))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 28))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); } } for (int ax2_fused_2 = 0; ax2_fused_2 < 2; ++ax2_fused_2) { for (int ax1 = 0; ax1 < 2; ++ax1) { barrier(CLK_LOCAL_MEM_FENCE); ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_intermediate_pad_rf_local_1[((ax1 * 2) + ax2_fused_2)]; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 16) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 8) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 4) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 2) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); } barrier(CLK_LOCAL_MEM_FENCE); NT_matmul_intermediate_pad_local[((ax1 * 2) + ax2_fused_2)] = ((volatile __local half*)red_buf0)[((convert_int(get_local_id(0))) * 32)]; } } for (int ax0 = 0; ax0 < 2; ++ax0) { for (int ax1_fused_2 = 0; ax1_fused_2 < 2; ++ax1_fused_2) { if (((bool)(((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)(((bool)(((convert_int(get_group_id(1))) - ((batch_size + 1) >> 1)) < 0)) || ((bool)((convert_int(get_group_id(1))) == 0)))))) && ((bool)((((convert_int(get_group_id(1))) * 2) + ax0) < batch_size))) { T_add[((((((convert_int(get_group_id(1))) * 4096) + (ax0 * 2048)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] = ((lv2[((((((convert_int(get_group_id(1))) * 4096) + (ax0 * 2048)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] + (NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_attention_dense_bias4[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)])) + input_embeds[((((((convert_int(get_group_id(1))) * 4096) + (ax0 * 2048)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]); } } } } // Function: gpt_neox_q4f16_1_chunk_lse_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size) { float in_thread_temp_max_shared[1]; __local float red_buf0[256]; __local float temp_max_shared[1]; float in_thread_temp_sum_shared[1]; __local float red_buf0_1[256]; __local float temp_sum_shared[1]; in_thread_temp_max_shared[0] = -3.402823e+38f; float condval; if ((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) < vocab_size)) { float condval_1; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_1 = (A[(((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0))))] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_1 = A[(((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0))))]; } condval = condval_1; } else { condval = -3.402823e+38f; } in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)condval); float condval_2; if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 256) < vocab_size)) { float condval_3; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_3 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 256)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_3 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 256)]; } condval_2 = condval_3; } else { condval_2 = -3.402823e+38f; } in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)condval_2); float condval_4; if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 512) < vocab_size)) { float condval_5; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_5 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 512)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_5 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 512)]; } condval_4 = condval_5; } else { condval_4 = -3.402823e+38f; } in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)condval_4); float condval_6; if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 768) < vocab_size)) { float condval_7; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_7 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 768)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_7 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 768)]; } condval_6 = condval_7; } else { condval_6 = -3.402823e+38f; } in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)condval_6); float condval_8; if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 1024) < vocab_size)) { float condval_9; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_9 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1024)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_9 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1024)]; } condval_8 = condval_9; } else { condval_8 = -3.402823e+38f; } in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)condval_8); float condval_10; if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 1280) < vocab_size)) { float condval_11; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_11 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1280)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_11 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1280)]; } condval_10 = condval_11; } else { condval_10 = -3.402823e+38f; } in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)condval_10); float condval_12; if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 1536) < vocab_size)) { float condval_13; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_13 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1536)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_13 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1536)]; } condval_12 = condval_13; } else { condval_12 = -3.402823e+38f; } in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)condval_12); float condval_14; if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 1792) < vocab_size)) { float condval_15; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_15 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1792)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_15 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1792)]; } condval_14 = condval_15; } else { condval_14 = -3.402823e+38f; } in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)condval_14); float condval_16; if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 2048) < vocab_size)) { float condval_17; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_17 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2048)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_17 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2048)]; } condval_16 = condval_17; } else { condval_16 = -3.402823e+38f; } in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)condval_16); float condval_18; if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 2304) < vocab_size)) { float condval_19; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_19 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2304)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_19 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2304)]; } condval_18 = condval_19; } else { condval_18 = -3.402823e+38f; } in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)condval_18); float condval_20; if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 2560) < vocab_size)) { float condval_21; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_21 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2560)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_21 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2560)]; } condval_20 = condval_21; } else { condval_20 = -3.402823e+38f; } in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)condval_20); float condval_22; if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 2816) < vocab_size)) { float condval_23; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_23 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2816)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_23 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2816)]; } condval_22 = condval_23; } else { condval_22 = -3.402823e+38f; } in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)condval_22); float condval_24; if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 3072) < vocab_size)) { float condval_25; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_25 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3072)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_25 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3072)]; } condval_24 = condval_25; } else { condval_24 = -3.402823e+38f; } in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)condval_24); float condval_26; if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 3328) < vocab_size)) { float condval_27; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_27 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3328)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_27 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3328)]; } condval_26 = condval_27; } else { condval_26 = -3.402823e+38f; } in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)condval_26); float condval_28; if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 3584) < vocab_size)) { float condval_29; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_29 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3584)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_29 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3584)]; } condval_28 = condval_29; } else { condval_28 = -3.402823e+38f; } in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)condval_28); float condval_30; if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 3840) < vocab_size)) { float condval_31; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_31 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3840)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_31 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3840)]; } condval_30 = condval_31; } else { condval_30 = -3.402823e+38f; } in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)condval_30); barrier(CLK_LOCAL_MEM_FENCE); ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = in_thread_temp_max_shared[0]; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 128) { ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = max((float)((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))], (float)((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 128)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 64) { ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = max((float)((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))], (float)((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 64)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 32) { ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = max((float)((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))], (float)((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 32)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 16) { ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = max((float)((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))], (float)((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 16)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 8) { ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = max((float)((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))], (float)((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 8)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 4) { ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = max((float)((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))], (float)((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 4)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 2) { ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = max((float)((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))], (float)((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 2)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 1) { ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = max((float)((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))], (float)((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 1)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) == 0) { temp_max_shared[0] = ((volatile __local float*)red_buf0)[0]; } in_thread_temp_sum_shared[0] = 0.000000e+00f; barrier(CLK_LOCAL_MEM_FENCE); float condval_32; if ((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) < vocab_size)) { float condval_33; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { float condval_34; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_34 = (A[(((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0))))] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_34 = A[(((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0))))]; } condval_33 = exp((condval_34 - temp_max_shared[0])); } else { float condval_35; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_35 = (A[(((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0))))] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_35 = A[(((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0))))]; } condval_33 = (convert_float((condval_35 == temp_max_shared[0]))); } condval_32 = condval_33; } else { condval_32 = 0.000000e+00f; } in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval_32); float condval_36; if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 256) < vocab_size)) { float condval_37; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { float condval_38; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_38 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 256)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_38 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 256)]; } condval_37 = exp((condval_38 - temp_max_shared[0])); } else { float condval_39; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_39 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 256)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_39 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 256)]; } condval_37 = (convert_float((condval_39 == temp_max_shared[0]))); } condval_36 = condval_37; } else { condval_36 = 0.000000e+00f; } in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval_36); float condval_40; if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 512) < vocab_size)) { float condval_41; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { float condval_42; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_42 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 512)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_42 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 512)]; } condval_41 = exp((condval_42 - temp_max_shared[0])); } else { float condval_43; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_43 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 512)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_43 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 512)]; } condval_41 = (convert_float((condval_43 == temp_max_shared[0]))); } condval_40 = condval_41; } else { condval_40 = 0.000000e+00f; } in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval_40); float condval_44; if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 768) < vocab_size)) { float condval_45; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { float condval_46; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_46 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 768)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_46 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 768)]; } condval_45 = exp((condval_46 - temp_max_shared[0])); } else { float condval_47; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_47 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 768)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_47 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 768)]; } condval_45 = (convert_float((condval_47 == temp_max_shared[0]))); } condval_44 = condval_45; } else { condval_44 = 0.000000e+00f; } in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval_44); float condval_48; if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 1024) < vocab_size)) { float condval_49; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { float condval_50; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_50 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1024)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_50 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1024)]; } condval_49 = exp((condval_50 - temp_max_shared[0])); } else { float condval_51; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_51 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1024)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_51 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1024)]; } condval_49 = (convert_float((condval_51 == temp_max_shared[0]))); } condval_48 = condval_49; } else { condval_48 = 0.000000e+00f; } in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval_48); float condval_52; if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 1280) < vocab_size)) { float condval_53; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { float condval_54; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_54 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1280)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_54 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1280)]; } condval_53 = exp((condval_54 - temp_max_shared[0])); } else { float condval_55; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_55 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1280)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_55 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1280)]; } condval_53 = (convert_float((condval_55 == temp_max_shared[0]))); } condval_52 = condval_53; } else { condval_52 = 0.000000e+00f; } in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval_52); float condval_56; if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 1536) < vocab_size)) { float condval_57; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { float condval_58; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_58 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1536)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_58 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1536)]; } condval_57 = exp((condval_58 - temp_max_shared[0])); } else { float condval_59; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_59 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1536)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_59 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1536)]; } condval_57 = (convert_float((condval_59 == temp_max_shared[0]))); } condval_56 = condval_57; } else { condval_56 = 0.000000e+00f; } in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval_56); float condval_60; if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 1792) < vocab_size)) { float condval_61; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { float condval_62; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_62 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1792)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_62 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1792)]; } condval_61 = exp((condval_62 - temp_max_shared[0])); } else { float condval_63; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_63 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1792)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_63 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1792)]; } condval_61 = (convert_float((condval_63 == temp_max_shared[0]))); } condval_60 = condval_61; } else { condval_60 = 0.000000e+00f; } in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval_60); float condval_64; if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 2048) < vocab_size)) { float condval_65; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { float condval_66; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_66 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2048)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_66 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2048)]; } condval_65 = exp((condval_66 - temp_max_shared[0])); } else { float condval_67; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_67 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2048)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_67 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2048)]; } condval_65 = (convert_float((condval_67 == temp_max_shared[0]))); } condval_64 = condval_65; } else { condval_64 = 0.000000e+00f; } in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval_64); float condval_68; if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 2304) < vocab_size)) { float condval_69; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { float condval_70; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_70 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2304)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_70 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2304)]; } condval_69 = exp((condval_70 - temp_max_shared[0])); } else { float condval_71; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_71 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2304)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_71 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2304)]; } condval_69 = (convert_float((condval_71 == temp_max_shared[0]))); } condval_68 = condval_69; } else { condval_68 = 0.000000e+00f; } in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval_68); float condval_72; if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 2560) < vocab_size)) { float condval_73; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { float condval_74; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_74 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2560)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_74 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2560)]; } condval_73 = exp((condval_74 - temp_max_shared[0])); } else { float condval_75; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_75 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2560)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_75 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2560)]; } condval_73 = (convert_float((condval_75 == temp_max_shared[0]))); } condval_72 = condval_73; } else { condval_72 = 0.000000e+00f; } in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval_72); float condval_76; if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 2816) < vocab_size)) { float condval_77; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { float condval_78; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_78 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2816)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_78 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2816)]; } condval_77 = exp((condval_78 - temp_max_shared[0])); } else { float condval_79; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_79 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2816)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_79 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2816)]; } condval_77 = (convert_float((condval_79 == temp_max_shared[0]))); } condval_76 = condval_77; } else { condval_76 = 0.000000e+00f; } in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval_76); float condval_80; if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 3072) < vocab_size)) { float condval_81; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { float condval_82; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_82 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3072)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_82 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3072)]; } condval_81 = exp((condval_82 - temp_max_shared[0])); } else { float condval_83; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_83 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3072)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_83 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3072)]; } condval_81 = (convert_float((condval_83 == temp_max_shared[0]))); } condval_80 = condval_81; } else { condval_80 = 0.000000e+00f; } in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval_80); float condval_84; if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 3328) < vocab_size)) { float condval_85; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { float condval_86; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_86 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3328)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_86 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3328)]; } condval_85 = exp((condval_86 - temp_max_shared[0])); } else { float condval_87; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_87 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3328)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_87 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3328)]; } condval_85 = (convert_float((condval_87 == temp_max_shared[0]))); } condval_84 = condval_85; } else { condval_84 = 0.000000e+00f; } in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval_84); float condval_88; if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 3584) < vocab_size)) { float condval_89; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { float condval_90; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_90 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3584)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_90 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3584)]; } condval_89 = exp((condval_90 - temp_max_shared[0])); } else { float condval_91; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_91 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3584)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_91 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3584)]; } condval_89 = (convert_float((condval_91 == temp_max_shared[0]))); } condval_88 = condval_89; } else { condval_88 = 0.000000e+00f; } in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval_88); float condval_92; if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + (convert_int(get_local_id(0)))) + 3840) < vocab_size)) { float condval_93; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { float condval_94; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_94 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3840)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_94 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3840)]; } condval_93 = exp((condval_94 - temp_max_shared[0])); } else { float condval_95; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_95 = (A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3840)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]); } else { condval_95 = A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3840)]; } condval_93 = (convert_float((condval_95 == temp_max_shared[0]))); } condval_92 = condval_93; } else { condval_92 = 0.000000e+00f; } in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval_92); barrier(CLK_LOCAL_MEM_FENCE); ((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] = in_thread_temp_sum_shared[0]; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 128) { ((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0_1)[((convert_int(get_local_id(0))) + 128)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 64) { ((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0_1)[((convert_int(get_local_id(0))) + 64)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 32) { ((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0_1)[((convert_int(get_local_id(0))) + 32)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 16) { ((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0_1)[((convert_int(get_local_id(0))) + 16)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 8) { ((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0_1)[((convert_int(get_local_id(0))) + 8)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 4) { ((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0_1)[((convert_int(get_local_id(0))) + 4)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 2) { ((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0_1)[((convert_int(get_local_id(0))) + 2)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 1) { ((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0_1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0_1)[((convert_int(get_local_id(0))) + 1)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) == 0) { temp_sum_shared[0] = ((volatile __local float*)red_buf0_1)[0]; } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 1) { float condval_96; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_96 = log(temp_sum_shared[0]); } else { condval_96 = temp_sum_shared[0]; } chunked_sum[(convert_int(get_group_id(0)))] = condval_96; chunked_max[(convert_int(get_group_id(0)))] = temp_max_shared[0]; } } // Function: gpt_neox_q4f16_1_reshape7_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len) { T_reshape[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] = reshape194[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]; } // Function: gpt_neox_q4f16_1_index_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len) { index[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] = layer_norm48[((((seq_len * 2048) + ((convert_int(get_group_id(0))) * 256)) + (convert_int(get_local_id(0)))) - 2048)]; } // Function: gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len) { float NT_matmul_intermediate_pad_rf_local[32]; half dequantize_local[16]; float NT_matmul_intermediate_pad_rf_local_1[4]; float NT_matmul_intermediate_pad_local[4]; __local float red_buf0[64]; for (int ax0_1_init = 0; ax0_1_init < 2; ++ax0_1_init) { for (int ax1_fused_2_init = 0; ax1_fused_2_init < 2; ++ax1_fused_2_init) { int8 v_ = (int8)((((ax0_1_init * 2) + ax1_fused_2_init))+(4*0), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*1), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*2), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*3), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*4), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*5), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*6), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*7)); float8 v__1 = (float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f); NT_matmul_intermediate_pad_rf_local[v_.s0] = v__1.s0; NT_matmul_intermediate_pad_rf_local[v_.s1] = v__1.s1; NT_matmul_intermediate_pad_rf_local[v_.s2] = v__1.s2; NT_matmul_intermediate_pad_rf_local[v_.s3] = v__1.s3; NT_matmul_intermediate_pad_rf_local[v_.s4] = v__1.s4; NT_matmul_intermediate_pad_rf_local[v_.s5] = v__1.s5; NT_matmul_intermediate_pad_rf_local[v_.s6] = v__1.s6; NT_matmul_intermediate_pad_rf_local[v_.s7] = v__1.s7; } } for (int ax2_fused_0 = 0; ax2_fused_0 < 32; ++ax2_fused_0) { for (int ax0_0 = 0; ax0_0 < 2; ++ax0_0) { dequantize_local[(ax0_0 * 8)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax0_0 * 1024)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)0) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 1)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax0_0 * 1024)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)4) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 2)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax0_0 * 1024)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)8) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 3)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax0_0 * 1024)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)12) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 4)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax0_0 * 1024)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)16) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 5)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax0_0 * 1024)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)20) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 6)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax0_0 * 1024)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)24) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 7)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax0_0 * 1024)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)28) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); } if (((convert_int(get_group_id(1))) * 2) < seq_len) { int8 v__2 = (int8)((0)+(4*0), (0)+(4*1), (0)+(4*2), (0)+(4*3), (0)+(4*4), (0)+(4*5), (0)+(4*6), (0)+(4*7)); float8 v__3 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__2.s0],NT_matmul_intermediate_pad_rf_local[v__2.s1],NT_matmul_intermediate_pad_rf_local[v__2.s2],NT_matmul_intermediate_pad_rf_local[v__2.s3],NT_matmul_intermediate_pad_rf_local[v__2.s4],NT_matmul_intermediate_pad_rf_local[v__2.s5],NT_matmul_intermediate_pad_rf_local[v__2.s6],NT_matmul_intermediate_pad_rf_local[v__2.s7])) + ((convert_float8(vload8(0, lv98 + ((((convert_int(get_group_id(1))) * 16384) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, dequantize_local + 0)))); NT_matmul_intermediate_pad_rf_local[v__2.s0] = v__3.s0; NT_matmul_intermediate_pad_rf_local[v__2.s1] = v__3.s1; NT_matmul_intermediate_pad_rf_local[v__2.s2] = v__3.s2; NT_matmul_intermediate_pad_rf_local[v__2.s3] = v__3.s3; NT_matmul_intermediate_pad_rf_local[v__2.s4] = v__3.s4; NT_matmul_intermediate_pad_rf_local[v__2.s5] = v__3.s5; NT_matmul_intermediate_pad_rf_local[v__2.s6] = v__3.s6; NT_matmul_intermediate_pad_rf_local[v__2.s7] = v__3.s7; int8 v__4 = (int8)((1)+(4*0), (1)+(4*1), (1)+(4*2), (1)+(4*3), (1)+(4*4), (1)+(4*5), (1)+(4*6), (1)+(4*7)); float8 v__5 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__4.s0],NT_matmul_intermediate_pad_rf_local[v__4.s1],NT_matmul_intermediate_pad_rf_local[v__4.s2],NT_matmul_intermediate_pad_rf_local[v__4.s3],NT_matmul_intermediate_pad_rf_local[v__4.s4],NT_matmul_intermediate_pad_rf_local[v__4.s5],NT_matmul_intermediate_pad_rf_local[v__4.s6],NT_matmul_intermediate_pad_rf_local[v__4.s7])) + ((convert_float8(vload8(0, lv98 + ((((convert_int(get_group_id(1))) * 16384) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, dequantize_local + 8)))); NT_matmul_intermediate_pad_rf_local[v__4.s0] = v__5.s0; NT_matmul_intermediate_pad_rf_local[v__4.s1] = v__5.s1; NT_matmul_intermediate_pad_rf_local[v__4.s2] = v__5.s2; NT_matmul_intermediate_pad_rf_local[v__4.s3] = v__5.s3; NT_matmul_intermediate_pad_rf_local[v__4.s4] = v__5.s4; NT_matmul_intermediate_pad_rf_local[v__4.s5] = v__5.s5; NT_matmul_intermediate_pad_rf_local[v__4.s6] = v__5.s6; NT_matmul_intermediate_pad_rf_local[v__4.s7] = v__5.s7; } if ((((convert_int(get_group_id(1))) * 2) + 1) < seq_len) { int8 v__6 = (int8)((2)+(4*0), (2)+(4*1), (2)+(4*2), (2)+(4*3), (2)+(4*4), (2)+(4*5), (2)+(4*6), (2)+(4*7)); float8 v__7 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__6.s0],NT_matmul_intermediate_pad_rf_local[v__6.s1],NT_matmul_intermediate_pad_rf_local[v__6.s2],NT_matmul_intermediate_pad_rf_local[v__6.s3],NT_matmul_intermediate_pad_rf_local[v__6.s4],NT_matmul_intermediate_pad_rf_local[v__6.s5],NT_matmul_intermediate_pad_rf_local[v__6.s6],NT_matmul_intermediate_pad_rf_local[v__6.s7])) + ((convert_float8(vload8(0, lv98 + (((((convert_int(get_group_id(1))) * 16384) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 8192)))) * (convert_float8(vload8(0, dequantize_local + 0)))); NT_matmul_intermediate_pad_rf_local[v__6.s0] = v__7.s0; NT_matmul_intermediate_pad_rf_local[v__6.s1] = v__7.s1; NT_matmul_intermediate_pad_rf_local[v__6.s2] = v__7.s2; NT_matmul_intermediate_pad_rf_local[v__6.s3] = v__7.s3; NT_matmul_intermediate_pad_rf_local[v__6.s4] = v__7.s4; NT_matmul_intermediate_pad_rf_local[v__6.s5] = v__7.s5; NT_matmul_intermediate_pad_rf_local[v__6.s6] = v__7.s6; NT_matmul_intermediate_pad_rf_local[v__6.s7] = v__7.s7; int8 v__8 = (int8)((3)+(4*0), (3)+(4*1), (3)+(4*2), (3)+(4*3), (3)+(4*4), (3)+(4*5), (3)+(4*6), (3)+(4*7)); float8 v__9 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__8.s0],NT_matmul_intermediate_pad_rf_local[v__8.s1],NT_matmul_intermediate_pad_rf_local[v__8.s2],NT_matmul_intermediate_pad_rf_local[v__8.s3],NT_matmul_intermediate_pad_rf_local[v__8.s4],NT_matmul_intermediate_pad_rf_local[v__8.s5],NT_matmul_intermediate_pad_rf_local[v__8.s6],NT_matmul_intermediate_pad_rf_local[v__8.s7])) + ((convert_float8(vload8(0, lv98 + (((((convert_int(get_group_id(1))) * 16384) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 8192)))) * (convert_float8(vload8(0, dequantize_local + 8)))); NT_matmul_intermediate_pad_rf_local[v__8.s0] = v__9.s0; NT_matmul_intermediate_pad_rf_local[v__8.s1] = v__9.s1; NT_matmul_intermediate_pad_rf_local[v__8.s2] = v__9.s2; NT_matmul_intermediate_pad_rf_local[v__8.s3] = v__9.s3; NT_matmul_intermediate_pad_rf_local[v__8.s4] = v__9.s4; NT_matmul_intermediate_pad_rf_local[v__8.s5] = v__9.s5; NT_matmul_intermediate_pad_rf_local[v__8.s6] = v__9.s6; NT_matmul_intermediate_pad_rf_local[v__8.s7] = v__9.s7; } } for (int ax3_fused_2_0 = 0; ax3_fused_2_0 < 1; ++ax3_fused_2_0) { for (int ax2 = 0; ax2 < 2; ++ax2) { vstore2(((float2)(0.000000e+00f, 0.000000e+00f)), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + (ax2 * 2))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 4))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 8))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 12))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 16))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 20))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 24))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 28))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); } } for (int ax2_fused_2 = 0; ax2_fused_2 < 2; ++ax2_fused_2) { for (int ax1 = 0; ax1 < 2; ++ax1) { barrier(CLK_LOCAL_MEM_FENCE); ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_intermediate_pad_rf_local_1[((ax1 * 2) + ax2_fused_2)]; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 16) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 8) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 4) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 2) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); } barrier(CLK_LOCAL_MEM_FENCE); NT_matmul_intermediate_pad_local[((ax1 * 2) + ax2_fused_2)] = ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) * 32)]; } } for (int ax0 = 0; ax0 < 2; ++ax0) { for (int ax1_fused_2 = 0; ax1_fused_2 < 2; ++ax1_fused_2) { if (((bool)(((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)(((bool)(((convert_int(get_group_id(1))) - ((seq_len + 1) >> 1)) < 0)) || ((bool)((convert_int(get_group_id(1))) == 0)))))) && ((bool)((((convert_int(get_group_id(1))) * 2) + ax0) < seq_len))) { compute[((((((convert_int(get_group_id(1))) * 4096) + (ax0 * 2048)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] = (convert_half((NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias3[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]))); } } } } // Function: gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len) { half NT_matmul_intermediate_pad_rf_local[32]; half dequantize_local[16]; half NT_matmul_intermediate_pad_rf_local_1[4]; half NT_matmul_intermediate_pad_local[4]; __local half red_buf0[64]; for (int ax0_1_init = 0; ax0_1_init < 2; ++ax0_1_init) { for (int ax1_fused_2_init = 0; ax1_fused_2_init < 2; ++ax1_fused_2_init) { int8 v_ = (int8)((((ax0_1_init * 2) + ax1_fused_2_init))+(4*0), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*1), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*2), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*3), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*4), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*5), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*6), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*7)); half8 v__1 = (half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f); NT_matmul_intermediate_pad_rf_local[v_.s0] = v__1.s0; NT_matmul_intermediate_pad_rf_local[v_.s1] = v__1.s1; NT_matmul_intermediate_pad_rf_local[v_.s2] = v__1.s2; NT_matmul_intermediate_pad_rf_local[v_.s3] = v__1.s3; NT_matmul_intermediate_pad_rf_local[v_.s4] = v__1.s4; NT_matmul_intermediate_pad_rf_local[v_.s5] = v__1.s5; NT_matmul_intermediate_pad_rf_local[v_.s6] = v__1.s6; NT_matmul_intermediate_pad_rf_local[v_.s7] = v__1.s7; } } for (int ax2_fused_0 = 0; ax2_fused_0 < 8; ++ax2_fused_0) { for (int ax0_0 = 0; ax0_0 < 2; ++ax0_0) { dequantize_local[(ax0_0 * 8)] = (((convert_half(((gpt_neox_layers_0_attention_dense_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)0) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_dense_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 1)] = (((convert_half(((gpt_neox_layers_0_attention_dense_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)4) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_dense_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 2)] = (((convert_half(((gpt_neox_layers_0_attention_dense_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)8) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_dense_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 3)] = (((convert_half(((gpt_neox_layers_0_attention_dense_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)12) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_dense_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 4)] = (((convert_half(((gpt_neox_layers_0_attention_dense_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)16) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_dense_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 5)] = (((convert_half(((gpt_neox_layers_0_attention_dense_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)20) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_dense_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 6)] = (((convert_half(((gpt_neox_layers_0_attention_dense_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)24) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_dense_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 7)] = (((convert_half(((gpt_neox_layers_0_attention_dense_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)28) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_dense_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); } if (((convert_int(get_group_id(1))) * 2) < seq_len) { int8 v__2 = (int8)((0)+(4*0), (0)+(4*1), (0)+(4*2), (0)+(4*3), (0)+(4*4), (0)+(4*5), (0)+(4*6), (0)+(4*7)); half8 v__3 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__2.s0],NT_matmul_intermediate_pad_rf_local[v__2.s1],NT_matmul_intermediate_pad_rf_local[v__2.s2],NT_matmul_intermediate_pad_rf_local[v__2.s3],NT_matmul_intermediate_pad_rf_local[v__2.s4],NT_matmul_intermediate_pad_rf_local[v__2.s5],NT_matmul_intermediate_pad_rf_local[v__2.s6],NT_matmul_intermediate_pad_rf_local[v__2.s7])) + (vload8(0, reshape195 + ((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, dequantize_local + 0)); NT_matmul_intermediate_pad_rf_local[v__2.s0] = v__3.s0; NT_matmul_intermediate_pad_rf_local[v__2.s1] = v__3.s1; NT_matmul_intermediate_pad_rf_local[v__2.s2] = v__3.s2; NT_matmul_intermediate_pad_rf_local[v__2.s3] = v__3.s3; NT_matmul_intermediate_pad_rf_local[v__2.s4] = v__3.s4; NT_matmul_intermediate_pad_rf_local[v__2.s5] = v__3.s5; NT_matmul_intermediate_pad_rf_local[v__2.s6] = v__3.s6; NT_matmul_intermediate_pad_rf_local[v__2.s7] = v__3.s7; int8 v__4 = (int8)((1)+(4*0), (1)+(4*1), (1)+(4*2), (1)+(4*3), (1)+(4*4), (1)+(4*5), (1)+(4*6), (1)+(4*7)); half8 v__5 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__4.s0],NT_matmul_intermediate_pad_rf_local[v__4.s1],NT_matmul_intermediate_pad_rf_local[v__4.s2],NT_matmul_intermediate_pad_rf_local[v__4.s3],NT_matmul_intermediate_pad_rf_local[v__4.s4],NT_matmul_intermediate_pad_rf_local[v__4.s5],NT_matmul_intermediate_pad_rf_local[v__4.s6],NT_matmul_intermediate_pad_rf_local[v__4.s7])) + (vload8(0, reshape195 + ((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, dequantize_local + 8)); NT_matmul_intermediate_pad_rf_local[v__4.s0] = v__5.s0; NT_matmul_intermediate_pad_rf_local[v__4.s1] = v__5.s1; NT_matmul_intermediate_pad_rf_local[v__4.s2] = v__5.s2; NT_matmul_intermediate_pad_rf_local[v__4.s3] = v__5.s3; NT_matmul_intermediate_pad_rf_local[v__4.s4] = v__5.s4; NT_matmul_intermediate_pad_rf_local[v__4.s5] = v__5.s5; NT_matmul_intermediate_pad_rf_local[v__4.s6] = v__5.s6; NT_matmul_intermediate_pad_rf_local[v__4.s7] = v__5.s7; } if ((((convert_int(get_group_id(1))) * 2) + 1) < seq_len) { int8 v__6 = (int8)((2)+(4*0), (2)+(4*1), (2)+(4*2), (2)+(4*3), (2)+(4*4), (2)+(4*5), (2)+(4*6), (2)+(4*7)); half8 v__7 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__6.s0],NT_matmul_intermediate_pad_rf_local[v__6.s1],NT_matmul_intermediate_pad_rf_local[v__6.s2],NT_matmul_intermediate_pad_rf_local[v__6.s3],NT_matmul_intermediate_pad_rf_local[v__6.s4],NT_matmul_intermediate_pad_rf_local[v__6.s5],NT_matmul_intermediate_pad_rf_local[v__6.s6],NT_matmul_intermediate_pad_rf_local[v__6.s7])) + (vload8(0, reshape195 + (((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)) * vload8(0, dequantize_local + 0)); NT_matmul_intermediate_pad_rf_local[v__6.s0] = v__7.s0; NT_matmul_intermediate_pad_rf_local[v__6.s1] = v__7.s1; NT_matmul_intermediate_pad_rf_local[v__6.s2] = v__7.s2; NT_matmul_intermediate_pad_rf_local[v__6.s3] = v__7.s3; NT_matmul_intermediate_pad_rf_local[v__6.s4] = v__7.s4; NT_matmul_intermediate_pad_rf_local[v__6.s5] = v__7.s5; NT_matmul_intermediate_pad_rf_local[v__6.s6] = v__7.s6; NT_matmul_intermediate_pad_rf_local[v__6.s7] = v__7.s7; int8 v__8 = (int8)((3)+(4*0), (3)+(4*1), (3)+(4*2), (3)+(4*3), (3)+(4*4), (3)+(4*5), (3)+(4*6), (3)+(4*7)); half8 v__9 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__8.s0],NT_matmul_intermediate_pad_rf_local[v__8.s1],NT_matmul_intermediate_pad_rf_local[v__8.s2],NT_matmul_intermediate_pad_rf_local[v__8.s3],NT_matmul_intermediate_pad_rf_local[v__8.s4],NT_matmul_intermediate_pad_rf_local[v__8.s5],NT_matmul_intermediate_pad_rf_local[v__8.s6],NT_matmul_intermediate_pad_rf_local[v__8.s7])) + (vload8(0, reshape195 + (((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)) * vload8(0, dequantize_local + 8)); NT_matmul_intermediate_pad_rf_local[v__8.s0] = v__9.s0; NT_matmul_intermediate_pad_rf_local[v__8.s1] = v__9.s1; NT_matmul_intermediate_pad_rf_local[v__8.s2] = v__9.s2; NT_matmul_intermediate_pad_rf_local[v__8.s3] = v__9.s3; NT_matmul_intermediate_pad_rf_local[v__8.s4] = v__9.s4; NT_matmul_intermediate_pad_rf_local[v__8.s5] = v__9.s5; NT_matmul_intermediate_pad_rf_local[v__8.s6] = v__9.s6; NT_matmul_intermediate_pad_rf_local[v__8.s7] = v__9.s7; } } for (int ax3_fused_2_0 = 0; ax3_fused_2_0 < 1; ++ax3_fused_2_0) { for (int ax2 = 0; ax2 < 2; ++ax2) { vstore2(((half2)((half)0.000000e+00f, (half)0.000000e+00f)), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + (ax2 * 2))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 4))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 8))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 12))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 16))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 20))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 24))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 28))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); } } for (int ax2_fused_2 = 0; ax2_fused_2 < 2; ++ax2_fused_2) { for (int ax1 = 0; ax1 < 2; ++ax1) { barrier(CLK_LOCAL_MEM_FENCE); ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_intermediate_pad_rf_local_1[((ax1 * 2) + ax2_fused_2)]; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 16) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 8) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 4) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 2) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); } barrier(CLK_LOCAL_MEM_FENCE); NT_matmul_intermediate_pad_local[((ax1 * 2) + ax2_fused_2)] = ((volatile __local half*)red_buf0)[((convert_int(get_local_id(0))) * 32)]; } } for (int ax0 = 0; ax0 < 2; ++ax0) { for (int ax1_fused_2 = 0; ax1_fused_2 < 2; ++ax1_fused_2) { if (((bool)(((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)(((bool)(((convert_int(get_group_id(1))) - ((seq_len + 1) >> 1)) < 0)) || ((bool)((convert_int(get_group_id(1))) == 0)))))) && ((bool)((((convert_int(get_group_id(1))) * 2) + ax0) < seq_len))) { T_add[((((((convert_int(get_group_id(1))) * 4096) + (ax0 * 2048)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] = ((lv74[((((((convert_int(get_group_id(1))) * 4096) + (ax0 * 2048)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] + (NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_attention_dense_bias3[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)])) + input_embeds[((((((convert_int(get_group_id(1))) * 4096) + (ax0 * 2048)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]); } } } } // Function: gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size) { float NT_matmul_intermediate_pad_rf_local[32]; half dequantize_local[16]; float NT_matmul_intermediate_pad_rf_local_1[4]; float NT_matmul_intermediate_pad_local[4]; __local float red_buf0[64]; for (int ax0_1_init = 0; ax0_1_init < 2; ++ax0_1_init) { for (int ax1_fused_2_init = 0; ax1_fused_2_init < 2; ++ax1_fused_2_init) { int8 v_ = (int8)((((ax0_1_init * 2) + ax1_fused_2_init))+(4*0), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*1), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*2), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*3), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*4), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*5), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*6), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*7)); float8 v__1 = (float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f); NT_matmul_intermediate_pad_rf_local[v_.s0] = v__1.s0; NT_matmul_intermediate_pad_rf_local[v_.s1] = v__1.s1; NT_matmul_intermediate_pad_rf_local[v_.s2] = v__1.s2; NT_matmul_intermediate_pad_rf_local[v_.s3] = v__1.s3; NT_matmul_intermediate_pad_rf_local[v_.s4] = v__1.s4; NT_matmul_intermediate_pad_rf_local[v_.s5] = v__1.s5; NT_matmul_intermediate_pad_rf_local[v_.s6] = v__1.s6; NT_matmul_intermediate_pad_rf_local[v_.s7] = v__1.s7; } } for (int ax2_fused_0 = 0; ax2_fused_0 < 32; ++ax2_fused_0) { for (int ax0_0 = 0; ax0_0 < 2; ++ax0_0) { dequantize_local[(ax0_0 * 8)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax0_0 * 1024)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)0) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 1)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax0_0 * 1024)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)4) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 2)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax0_0 * 1024)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)8) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 3)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax0_0 * 1024)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)12) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 4)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax0_0 * 1024)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)16) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 5)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax0_0 * 1024)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)20) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 6)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax0_0 * 1024)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)24) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 7)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax0_0 * 1024)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)28) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); } if (((convert_int(get_group_id(1))) * 2) < batch_size) { int8 v__2 = (int8)((0)+(4*0), (0)+(4*1), (0)+(4*2), (0)+(4*3), (0)+(4*4), (0)+(4*5), (0)+(4*6), (0)+(4*7)); float8 v__3 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__2.s0],NT_matmul_intermediate_pad_rf_local[v__2.s1],NT_matmul_intermediate_pad_rf_local[v__2.s2],NT_matmul_intermediate_pad_rf_local[v__2.s3],NT_matmul_intermediate_pad_rf_local[v__2.s4],NT_matmul_intermediate_pad_rf_local[v__2.s5],NT_matmul_intermediate_pad_rf_local[v__2.s6],NT_matmul_intermediate_pad_rf_local[v__2.s7])) + ((convert_float8(vload8(0, lv1 + ((((convert_int(get_group_id(1))) * 16384) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, dequantize_local + 0)))); NT_matmul_intermediate_pad_rf_local[v__2.s0] = v__3.s0; NT_matmul_intermediate_pad_rf_local[v__2.s1] = v__3.s1; NT_matmul_intermediate_pad_rf_local[v__2.s2] = v__3.s2; NT_matmul_intermediate_pad_rf_local[v__2.s3] = v__3.s3; NT_matmul_intermediate_pad_rf_local[v__2.s4] = v__3.s4; NT_matmul_intermediate_pad_rf_local[v__2.s5] = v__3.s5; NT_matmul_intermediate_pad_rf_local[v__2.s6] = v__3.s6; NT_matmul_intermediate_pad_rf_local[v__2.s7] = v__3.s7; int8 v__4 = (int8)((1)+(4*0), (1)+(4*1), (1)+(4*2), (1)+(4*3), (1)+(4*4), (1)+(4*5), (1)+(4*6), (1)+(4*7)); float8 v__5 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__4.s0],NT_matmul_intermediate_pad_rf_local[v__4.s1],NT_matmul_intermediate_pad_rf_local[v__4.s2],NT_matmul_intermediate_pad_rf_local[v__4.s3],NT_matmul_intermediate_pad_rf_local[v__4.s4],NT_matmul_intermediate_pad_rf_local[v__4.s5],NT_matmul_intermediate_pad_rf_local[v__4.s6],NT_matmul_intermediate_pad_rf_local[v__4.s7])) + ((convert_float8(vload8(0, lv1 + ((((convert_int(get_group_id(1))) * 16384) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, dequantize_local + 8)))); NT_matmul_intermediate_pad_rf_local[v__4.s0] = v__5.s0; NT_matmul_intermediate_pad_rf_local[v__4.s1] = v__5.s1; NT_matmul_intermediate_pad_rf_local[v__4.s2] = v__5.s2; NT_matmul_intermediate_pad_rf_local[v__4.s3] = v__5.s3; NT_matmul_intermediate_pad_rf_local[v__4.s4] = v__5.s4; NT_matmul_intermediate_pad_rf_local[v__4.s5] = v__5.s5; NT_matmul_intermediate_pad_rf_local[v__4.s6] = v__5.s6; NT_matmul_intermediate_pad_rf_local[v__4.s7] = v__5.s7; } if ((((convert_int(get_group_id(1))) * 2) + 1) < batch_size) { int8 v__6 = (int8)((2)+(4*0), (2)+(4*1), (2)+(4*2), (2)+(4*3), (2)+(4*4), (2)+(4*5), (2)+(4*6), (2)+(4*7)); float8 v__7 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__6.s0],NT_matmul_intermediate_pad_rf_local[v__6.s1],NT_matmul_intermediate_pad_rf_local[v__6.s2],NT_matmul_intermediate_pad_rf_local[v__6.s3],NT_matmul_intermediate_pad_rf_local[v__6.s4],NT_matmul_intermediate_pad_rf_local[v__6.s5],NT_matmul_intermediate_pad_rf_local[v__6.s6],NT_matmul_intermediate_pad_rf_local[v__6.s7])) + ((convert_float8(vload8(0, lv1 + (((((convert_int(get_group_id(1))) * 16384) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 8192)))) * (convert_float8(vload8(0, dequantize_local + 0)))); NT_matmul_intermediate_pad_rf_local[v__6.s0] = v__7.s0; NT_matmul_intermediate_pad_rf_local[v__6.s1] = v__7.s1; NT_matmul_intermediate_pad_rf_local[v__6.s2] = v__7.s2; NT_matmul_intermediate_pad_rf_local[v__6.s3] = v__7.s3; NT_matmul_intermediate_pad_rf_local[v__6.s4] = v__7.s4; NT_matmul_intermediate_pad_rf_local[v__6.s5] = v__7.s5; NT_matmul_intermediate_pad_rf_local[v__6.s6] = v__7.s6; NT_matmul_intermediate_pad_rf_local[v__6.s7] = v__7.s7; int8 v__8 = (int8)((3)+(4*0), (3)+(4*1), (3)+(4*2), (3)+(4*3), (3)+(4*4), (3)+(4*5), (3)+(4*6), (3)+(4*7)); float8 v__9 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__8.s0],NT_matmul_intermediate_pad_rf_local[v__8.s1],NT_matmul_intermediate_pad_rf_local[v__8.s2],NT_matmul_intermediate_pad_rf_local[v__8.s3],NT_matmul_intermediate_pad_rf_local[v__8.s4],NT_matmul_intermediate_pad_rf_local[v__8.s5],NT_matmul_intermediate_pad_rf_local[v__8.s6],NT_matmul_intermediate_pad_rf_local[v__8.s7])) + ((convert_float8(vload8(0, lv1 + (((((convert_int(get_group_id(1))) * 16384) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 8192)))) * (convert_float8(vload8(0, dequantize_local + 8)))); NT_matmul_intermediate_pad_rf_local[v__8.s0] = v__9.s0; NT_matmul_intermediate_pad_rf_local[v__8.s1] = v__9.s1; NT_matmul_intermediate_pad_rf_local[v__8.s2] = v__9.s2; NT_matmul_intermediate_pad_rf_local[v__8.s3] = v__9.s3; NT_matmul_intermediate_pad_rf_local[v__8.s4] = v__9.s4; NT_matmul_intermediate_pad_rf_local[v__8.s5] = v__9.s5; NT_matmul_intermediate_pad_rf_local[v__8.s6] = v__9.s6; NT_matmul_intermediate_pad_rf_local[v__8.s7] = v__9.s7; } } for (int ax3_fused_2_0 = 0; ax3_fused_2_0 < 1; ++ax3_fused_2_0) { for (int ax2 = 0; ax2 < 2; ++ax2) { vstore2(((float2)(0.000000e+00f, 0.000000e+00f)), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + (ax2 * 2))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 4))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 8))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 12))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 16))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 20))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 24))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 28))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); } } for (int ax2_fused_2 = 0; ax2_fused_2 < 2; ++ax2_fused_2) { for (int ax1 = 0; ax1 < 2; ++ax1) { barrier(CLK_LOCAL_MEM_FENCE); ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_intermediate_pad_rf_local_1[((ax1 * 2) + ax2_fused_2)]; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 16) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 8) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 4) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 2) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); } barrier(CLK_LOCAL_MEM_FENCE); NT_matmul_intermediate_pad_local[((ax1 * 2) + ax2_fused_2)] = ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) * 32)]; } } for (int ax0 = 0; ax0 < 2; ++ax0) { for (int ax1_fused_2 = 0; ax1_fused_2 < 2; ++ax1_fused_2) { if (((bool)(((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)(((bool)(((convert_int(get_group_id(1))) - ((batch_size + 1) >> 1)) < 0)) || ((bool)((convert_int(get_group_id(1))) == 0)))))) && ((bool)((((convert_int(get_group_id(1))) * 2) + ax0) < batch_size))) { compute[((((((convert_int(get_group_id(1))) * 4096) + (ax0 * 2048)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] = (convert_half((NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias4[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]))); } } } } // Function: gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49) { half NT_matmul_rf_local[8]; uint gpt_neox_layers_0_attention_query_key_value_q_weight2_local[1]; half NT_matmul_rf_local_1[1]; __local half red_buf0[64]; vstore8(((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, NT_matmul_rf_local + 0); for (int ax1_0_fused_ax1_1_fused_0 = 0; ax1_0_fused_ax1_1_fused_0 < 8; ++ax1_0_fused_ax1_1_fused_0) { gpt_neox_layers_0_attention_query_key_value_q_weight2_local[0] = gpt_neox_layers_0_attention_query_key_value_q_weight2[(((((convert_int(get_group_id(0))) * 512) + ((convert_int(get_local_id(0))) * 256)) + (ax1_0_fused_ax1_1_fused_0 * 32)) + (convert_int(get_local_id(1))))]; vstore8((vload8(0, NT_matmul_rf_local + 0) + (vload8(0, layer_norm49 + ((ax1_0_fused_ax1_1_fused_0 * 256) + ((convert_int(get_local_id(1))) * 8))) * (((convert_half8(((((uint8)(gpt_neox_layers_0_attention_query_key_value_q_weight2_local[0], gpt_neox_layers_0_attention_query_key_value_q_weight2_local[0], gpt_neox_layers_0_attention_query_key_value_q_weight2_local[0], gpt_neox_layers_0_attention_query_key_value_q_weight2_local[0], gpt_neox_layers_0_attention_query_key_value_q_weight2_local[0], gpt_neox_layers_0_attention_query_key_value_q_weight2_local[0], gpt_neox_layers_0_attention_query_key_value_q_weight2_local[0], gpt_neox_layers_0_attention_query_key_value_q_weight2_local[0])) >> ((uint8)(((uint)0)+((uint)4*0), ((uint)0)+((uint)4*1), ((uint)0)+((uint)4*2), ((uint)0)+((uint)4*3), ((uint)0)+((uint)4*4), ((uint)0)+((uint)4*5), ((uint)0)+((uint)4*6), ((uint)0)+((uint)4*7)))) & ((uint8)((uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15))))) - ((half8)((half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f))) * ((half8)(gpt_neox_layers_0_attention_query_key_value_q_scale2[(((((convert_int(get_group_id(0))) * 128) + ((convert_int(get_local_id(0))) * 64)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], gpt_neox_layers_0_attention_query_key_value_q_scale2[(((((convert_int(get_group_id(0))) * 128) + ((convert_int(get_local_id(0))) * 64)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], gpt_neox_layers_0_attention_query_key_value_q_scale2[(((((convert_int(get_group_id(0))) * 128) + ((convert_int(get_local_id(0))) * 64)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], gpt_neox_layers_0_attention_query_key_value_q_scale2[(((((convert_int(get_group_id(0))) * 128) + ((convert_int(get_local_id(0))) * 64)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], gpt_neox_layers_0_attention_query_key_value_q_scale2[(((((convert_int(get_group_id(0))) * 128) + ((convert_int(get_local_id(0))) * 64)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], gpt_neox_layers_0_attention_query_key_value_q_scale2[(((((convert_int(get_group_id(0))) * 128) + ((convert_int(get_local_id(0))) * 64)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], gpt_neox_layers_0_attention_query_key_value_q_scale2[(((((convert_int(get_group_id(0))) * 128) + ((convert_int(get_local_id(0))) * 64)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], gpt_neox_layers_0_attention_query_key_value_q_scale2[(((((convert_int(get_group_id(0))) * 128) + ((convert_int(get_local_id(0))) * 64)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]))))), 0, NT_matmul_rf_local + 0); } for (int ax2_fused_2_0 = 0; ax2_fused_2_0 < 1; ++ax2_fused_2_0) { NT_matmul_rf_local_1[0] = (half)0.000000e+00f; NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[0]); NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[1]); NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[2]); NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[3]); NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[4]); NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[5]); NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[6]); NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[7]); } barrier(CLK_LOCAL_MEM_FENCE); ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_rf_local_1[0]; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 16) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 8) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 4) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 2) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) == 0) { T_add[(((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0))))] = (((volatile __local half*)red_buf0)[((convert_int(get_local_id(0))) * 32)] + gpt_neox_layers_0_attention_query_key_value_bias2[(((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0))))]); } } // Function: gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size) { float NT_matmul_intermediate_pad_rf_local[32]; half dequantize_local[16]; float NT_matmul_intermediate_pad_rf_local_1[4]; float NT_matmul_intermediate_pad_local[4]; __local float red_buf0[64]; for (int ax0_1_init = 0; ax0_1_init < 2; ++ax0_1_init) { for (int ax1_fused_2_init = 0; ax1_fused_2_init < 2; ++ax1_fused_2_init) { int8 v_ = (int8)((((ax0_1_init * 2) + ax1_fused_2_init))+(4*0), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*1), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*2), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*3), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*4), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*5), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*6), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*7)); float8 v__1 = (float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f); NT_matmul_intermediate_pad_rf_local[v_.s0] = v__1.s0; NT_matmul_intermediate_pad_rf_local[v_.s1] = v__1.s1; NT_matmul_intermediate_pad_rf_local[v_.s2] = v__1.s2; NT_matmul_intermediate_pad_rf_local[v_.s3] = v__1.s3; NT_matmul_intermediate_pad_rf_local[v_.s4] = v__1.s4; NT_matmul_intermediate_pad_rf_local[v_.s5] = v__1.s5; NT_matmul_intermediate_pad_rf_local[v_.s6] = v__1.s6; NT_matmul_intermediate_pad_rf_local[v_.s7] = v__1.s7; } } for (int ax2_fused_0 = 0; ax2_fused_0 < 8; ++ax2_fused_0) { for (int ax0_0 = 0; ax0_0 < 2; ++ax0_0) { dequantize_local[(ax0_0 * 8)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)0) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 1)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)4) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 2)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)8) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 3)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)12) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 4)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)16) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 5)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)20) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 6)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)24) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 7)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)28) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); } if (((convert_int(get_group_id(1))) * 2) < batch_size) { int8 v__2 = (int8)((0)+(4*0), (0)+(4*1), (0)+(4*2), (0)+(4*3), (0)+(4*4), (0)+(4*5), (0)+(4*6), (0)+(4*7)); float8 v__3 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__2.s0],NT_matmul_intermediate_pad_rf_local[v__2.s1],NT_matmul_intermediate_pad_rf_local[v__2.s2],NT_matmul_intermediate_pad_rf_local[v__2.s3],NT_matmul_intermediate_pad_rf_local[v__2.s4],NT_matmul_intermediate_pad_rf_local[v__2.s5],NT_matmul_intermediate_pad_rf_local[v__2.s6],NT_matmul_intermediate_pad_rf_local[v__2.s7])) + ((convert_float8(vload8(0, layer_norm148 + ((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, dequantize_local + 0)))); NT_matmul_intermediate_pad_rf_local[v__2.s0] = v__3.s0; NT_matmul_intermediate_pad_rf_local[v__2.s1] = v__3.s1; NT_matmul_intermediate_pad_rf_local[v__2.s2] = v__3.s2; NT_matmul_intermediate_pad_rf_local[v__2.s3] = v__3.s3; NT_matmul_intermediate_pad_rf_local[v__2.s4] = v__3.s4; NT_matmul_intermediate_pad_rf_local[v__2.s5] = v__3.s5; NT_matmul_intermediate_pad_rf_local[v__2.s6] = v__3.s6; NT_matmul_intermediate_pad_rf_local[v__2.s7] = v__3.s7; int8 v__4 = (int8)((1)+(4*0), (1)+(4*1), (1)+(4*2), (1)+(4*3), (1)+(4*4), (1)+(4*5), (1)+(4*6), (1)+(4*7)); float8 v__5 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__4.s0],NT_matmul_intermediate_pad_rf_local[v__4.s1],NT_matmul_intermediate_pad_rf_local[v__4.s2],NT_matmul_intermediate_pad_rf_local[v__4.s3],NT_matmul_intermediate_pad_rf_local[v__4.s4],NT_matmul_intermediate_pad_rf_local[v__4.s5],NT_matmul_intermediate_pad_rf_local[v__4.s6],NT_matmul_intermediate_pad_rf_local[v__4.s7])) + ((convert_float8(vload8(0, layer_norm148 + ((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, dequantize_local + 8)))); NT_matmul_intermediate_pad_rf_local[v__4.s0] = v__5.s0; NT_matmul_intermediate_pad_rf_local[v__4.s1] = v__5.s1; NT_matmul_intermediate_pad_rf_local[v__4.s2] = v__5.s2; NT_matmul_intermediate_pad_rf_local[v__4.s3] = v__5.s3; NT_matmul_intermediate_pad_rf_local[v__4.s4] = v__5.s4; NT_matmul_intermediate_pad_rf_local[v__4.s5] = v__5.s5; NT_matmul_intermediate_pad_rf_local[v__4.s6] = v__5.s6; NT_matmul_intermediate_pad_rf_local[v__4.s7] = v__5.s7; } if ((((convert_int(get_group_id(1))) * 2) + 1) < batch_size) { int8 v__6 = (int8)((2)+(4*0), (2)+(4*1), (2)+(4*2), (2)+(4*3), (2)+(4*4), (2)+(4*5), (2)+(4*6), (2)+(4*7)); float8 v__7 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__6.s0],NT_matmul_intermediate_pad_rf_local[v__6.s1],NT_matmul_intermediate_pad_rf_local[v__6.s2],NT_matmul_intermediate_pad_rf_local[v__6.s3],NT_matmul_intermediate_pad_rf_local[v__6.s4],NT_matmul_intermediate_pad_rf_local[v__6.s5],NT_matmul_intermediate_pad_rf_local[v__6.s6],NT_matmul_intermediate_pad_rf_local[v__6.s7])) + ((convert_float8(vload8(0, layer_norm148 + (((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)))) * (convert_float8(vload8(0, dequantize_local + 0)))); NT_matmul_intermediate_pad_rf_local[v__6.s0] = v__7.s0; NT_matmul_intermediate_pad_rf_local[v__6.s1] = v__7.s1; NT_matmul_intermediate_pad_rf_local[v__6.s2] = v__7.s2; NT_matmul_intermediate_pad_rf_local[v__6.s3] = v__7.s3; NT_matmul_intermediate_pad_rf_local[v__6.s4] = v__7.s4; NT_matmul_intermediate_pad_rf_local[v__6.s5] = v__7.s5; NT_matmul_intermediate_pad_rf_local[v__6.s6] = v__7.s6; NT_matmul_intermediate_pad_rf_local[v__6.s7] = v__7.s7; int8 v__8 = (int8)((3)+(4*0), (3)+(4*1), (3)+(4*2), (3)+(4*3), (3)+(4*4), (3)+(4*5), (3)+(4*6), (3)+(4*7)); float8 v__9 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__8.s0],NT_matmul_intermediate_pad_rf_local[v__8.s1],NT_matmul_intermediate_pad_rf_local[v__8.s2],NT_matmul_intermediate_pad_rf_local[v__8.s3],NT_matmul_intermediate_pad_rf_local[v__8.s4],NT_matmul_intermediate_pad_rf_local[v__8.s5],NT_matmul_intermediate_pad_rf_local[v__8.s6],NT_matmul_intermediate_pad_rf_local[v__8.s7])) + ((convert_float8(vload8(0, layer_norm148 + (((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)))) * (convert_float8(vload8(0, dequantize_local + 8)))); NT_matmul_intermediate_pad_rf_local[v__8.s0] = v__9.s0; NT_matmul_intermediate_pad_rf_local[v__8.s1] = v__9.s1; NT_matmul_intermediate_pad_rf_local[v__8.s2] = v__9.s2; NT_matmul_intermediate_pad_rf_local[v__8.s3] = v__9.s3; NT_matmul_intermediate_pad_rf_local[v__8.s4] = v__9.s4; NT_matmul_intermediate_pad_rf_local[v__8.s5] = v__9.s5; NT_matmul_intermediate_pad_rf_local[v__8.s6] = v__9.s6; NT_matmul_intermediate_pad_rf_local[v__8.s7] = v__9.s7; } } for (int ax3_fused_2_0 = 0; ax3_fused_2_0 < 1; ++ax3_fused_2_0) { for (int ax2 = 0; ax2 < 2; ++ax2) { vstore2(((float2)(0.000000e+00f, 0.000000e+00f)), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + (ax2 * 2))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 4))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 8))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 12))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 16))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 20))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 24))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 28))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); } } for (int ax2_fused_2 = 0; ax2_fused_2 < 2; ++ax2_fused_2) { for (int ax1 = 0; ax1 < 2; ++ax1) { barrier(CLK_LOCAL_MEM_FENCE); ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_intermediate_pad_rf_local_1[((ax1 * 2) + ax2_fused_2)]; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 16) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 8) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 4) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 2) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); } barrier(CLK_LOCAL_MEM_FENCE); NT_matmul_intermediate_pad_local[((ax1 * 2) + ax2_fused_2)] = ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) * 32)]; } } for (int ax0 = 0; ax0 < 2; ++ax0) { for (int ax1_fused_2 = 0; ax1_fused_2 < 2; ++ax1_fused_2) { if (((bool)(((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)(((bool)(((convert_int(get_group_id(1))) - ((batch_size + 1) >> 1)) < 0)) || ((bool)((convert_int(get_group_id(1))) == 0)))))) && ((bool)((((convert_int(get_group_id(1))) * 2) + ax0) < batch_size))) { compute[((((((convert_int(get_group_id(1))) * 16384) + (ax0 * 8192)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] = (convert_half(((NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]) * 7.071068e-01f)) * 5.000000e-01f))))); } } } } // Function: gpt_neox_q4f16_1_fused_dequantize_take1_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size) { T_take[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] = (((convert_half(((gpt_neox_embed_in_q_weight[(((input_ids[((convert_int(get_group_id(0))) >> 3)] * 256) + (((convert_int(get_group_id(0))) & 7) * 32)) + ((convert_int(get_local_id(0))) >> 3))] >> (convert_uint((((convert_int(get_local_id(0))) & 7) * 4)))) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_embed_in_q_scale[(((input_ids[((convert_int(get_group_id(0))) >> 3)] * 64) + (((convert_int(get_group_id(0))) & 7) * 8)) + ((convert_int(get_local_id(0))) >> 5))]); } // Function: gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1 #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len) { float NT_matmul_intermediate_pad_rf_local[64]; half dequantize_local[16]; float NT_matmul_intermediate_pad_rf_local_1[8]; float NT_matmul_intermediate_pad_local[8]; __local float red_buf0[64]; for (int ax0_1_init = 0; ax0_1_init < 4; ++ax0_1_init) { for (int ax1_fused_2_init = 0; ax1_fused_2_init < 2; ++ax1_fused_2_init) { int8 v_ = (int8)((((ax0_1_init * 2) + ax1_fused_2_init))+(8*0), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*1), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*2), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*3), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*4), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*5), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*6), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*7)); float8 v__1 = (float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f); NT_matmul_intermediate_pad_rf_local[v_.s0] = v__1.s0; NT_matmul_intermediate_pad_rf_local[v_.s1] = v__1.s1; NT_matmul_intermediate_pad_rf_local[v_.s2] = v__1.s2; NT_matmul_intermediate_pad_rf_local[v_.s3] = v__1.s3; NT_matmul_intermediate_pad_rf_local[v_.s4] = v__1.s4; NT_matmul_intermediate_pad_rf_local[v_.s5] = v__1.s5; NT_matmul_intermediate_pad_rf_local[v_.s6] = v__1.s6; NT_matmul_intermediate_pad_rf_local[v_.s7] = v__1.s7; } } for (int ax2_fused_0 = 0; ax2_fused_0 < 8; ++ax2_fused_0) { for (int ax0_0 = 0; ax0_0 < 2; ++ax0_0) { dequantize_local[(ax0_0 * 8)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)0) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 1)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)4) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 2)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)8) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 3)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)12) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 4)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)16) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 5)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)20) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 6)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)24) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 7)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)28) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); } if (((convert_int(get_group_id(1))) * 4) < seq_len) { int8 v__2 = (int8)((0)+(8*0), (0)+(8*1), (0)+(8*2), (0)+(8*3), (0)+(8*4), (0)+(8*5), (0)+(8*6), (0)+(8*7)); float8 v__3 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__2.s0],NT_matmul_intermediate_pad_rf_local[v__2.s1],NT_matmul_intermediate_pad_rf_local[v__2.s2],NT_matmul_intermediate_pad_rf_local[v__2.s3],NT_matmul_intermediate_pad_rf_local[v__2.s4],NT_matmul_intermediate_pad_rf_local[v__2.s5],NT_matmul_intermediate_pad_rf_local[v__2.s6],NT_matmul_intermediate_pad_rf_local[v__2.s7])) + ((convert_float8(vload8(0, layer_norm99 + ((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, dequantize_local + 0)))); NT_matmul_intermediate_pad_rf_local[v__2.s0] = v__3.s0; NT_matmul_intermediate_pad_rf_local[v__2.s1] = v__3.s1; NT_matmul_intermediate_pad_rf_local[v__2.s2] = v__3.s2; NT_matmul_intermediate_pad_rf_local[v__2.s3] = v__3.s3; NT_matmul_intermediate_pad_rf_local[v__2.s4] = v__3.s4; NT_matmul_intermediate_pad_rf_local[v__2.s5] = v__3.s5; NT_matmul_intermediate_pad_rf_local[v__2.s6] = v__3.s6; NT_matmul_intermediate_pad_rf_local[v__2.s7] = v__3.s7; int8 v__4 = (int8)((1)+(8*0), (1)+(8*1), (1)+(8*2), (1)+(8*3), (1)+(8*4), (1)+(8*5), (1)+(8*6), (1)+(8*7)); float8 v__5 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__4.s0],NT_matmul_intermediate_pad_rf_local[v__4.s1],NT_matmul_intermediate_pad_rf_local[v__4.s2],NT_matmul_intermediate_pad_rf_local[v__4.s3],NT_matmul_intermediate_pad_rf_local[v__4.s4],NT_matmul_intermediate_pad_rf_local[v__4.s5],NT_matmul_intermediate_pad_rf_local[v__4.s6],NT_matmul_intermediate_pad_rf_local[v__4.s7])) + ((convert_float8(vload8(0, layer_norm99 + ((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, dequantize_local + 8)))); NT_matmul_intermediate_pad_rf_local[v__4.s0] = v__5.s0; NT_matmul_intermediate_pad_rf_local[v__4.s1] = v__5.s1; NT_matmul_intermediate_pad_rf_local[v__4.s2] = v__5.s2; NT_matmul_intermediate_pad_rf_local[v__4.s3] = v__5.s3; NT_matmul_intermediate_pad_rf_local[v__4.s4] = v__5.s4; NT_matmul_intermediate_pad_rf_local[v__4.s5] = v__5.s5; NT_matmul_intermediate_pad_rf_local[v__4.s6] = v__5.s6; NT_matmul_intermediate_pad_rf_local[v__4.s7] = v__5.s7; } if ((((convert_int(get_group_id(1))) * 4) + 1) < seq_len) { int8 v__6 = (int8)((2)+(8*0), (2)+(8*1), (2)+(8*2), (2)+(8*3), (2)+(8*4), (2)+(8*5), (2)+(8*6), (2)+(8*7)); float8 v__7 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__6.s0],NT_matmul_intermediate_pad_rf_local[v__6.s1],NT_matmul_intermediate_pad_rf_local[v__6.s2],NT_matmul_intermediate_pad_rf_local[v__6.s3],NT_matmul_intermediate_pad_rf_local[v__6.s4],NT_matmul_intermediate_pad_rf_local[v__6.s5],NT_matmul_intermediate_pad_rf_local[v__6.s6],NT_matmul_intermediate_pad_rf_local[v__6.s7])) + ((convert_float8(vload8(0, layer_norm99 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)))) * (convert_float8(vload8(0, dequantize_local + 0)))); NT_matmul_intermediate_pad_rf_local[v__6.s0] = v__7.s0; NT_matmul_intermediate_pad_rf_local[v__6.s1] = v__7.s1; NT_matmul_intermediate_pad_rf_local[v__6.s2] = v__7.s2; NT_matmul_intermediate_pad_rf_local[v__6.s3] = v__7.s3; NT_matmul_intermediate_pad_rf_local[v__6.s4] = v__7.s4; NT_matmul_intermediate_pad_rf_local[v__6.s5] = v__7.s5; NT_matmul_intermediate_pad_rf_local[v__6.s6] = v__7.s6; NT_matmul_intermediate_pad_rf_local[v__6.s7] = v__7.s7; int8 v__8 = (int8)((3)+(8*0), (3)+(8*1), (3)+(8*2), (3)+(8*3), (3)+(8*4), (3)+(8*5), (3)+(8*6), (3)+(8*7)); float8 v__9 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__8.s0],NT_matmul_intermediate_pad_rf_local[v__8.s1],NT_matmul_intermediate_pad_rf_local[v__8.s2],NT_matmul_intermediate_pad_rf_local[v__8.s3],NT_matmul_intermediate_pad_rf_local[v__8.s4],NT_matmul_intermediate_pad_rf_local[v__8.s5],NT_matmul_intermediate_pad_rf_local[v__8.s6],NT_matmul_intermediate_pad_rf_local[v__8.s7])) + ((convert_float8(vload8(0, layer_norm99 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)))) * (convert_float8(vload8(0, dequantize_local + 8)))); NT_matmul_intermediate_pad_rf_local[v__8.s0] = v__9.s0; NT_matmul_intermediate_pad_rf_local[v__8.s1] = v__9.s1; NT_matmul_intermediate_pad_rf_local[v__8.s2] = v__9.s2; NT_matmul_intermediate_pad_rf_local[v__8.s3] = v__9.s3; NT_matmul_intermediate_pad_rf_local[v__8.s4] = v__9.s4; NT_matmul_intermediate_pad_rf_local[v__8.s5] = v__9.s5; NT_matmul_intermediate_pad_rf_local[v__8.s6] = v__9.s6; NT_matmul_intermediate_pad_rf_local[v__8.s7] = v__9.s7; } if ((((convert_int(get_group_id(1))) * 4) + 2) < seq_len) { int8 v__10 = (int8)((4)+(8*0), (4)+(8*1), (4)+(8*2), (4)+(8*3), (4)+(8*4), (4)+(8*5), (4)+(8*6), (4)+(8*7)); float8 v__11 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__10.s0],NT_matmul_intermediate_pad_rf_local[v__10.s1],NT_matmul_intermediate_pad_rf_local[v__10.s2],NT_matmul_intermediate_pad_rf_local[v__10.s3],NT_matmul_intermediate_pad_rf_local[v__10.s4],NT_matmul_intermediate_pad_rf_local[v__10.s5],NT_matmul_intermediate_pad_rf_local[v__10.s6],NT_matmul_intermediate_pad_rf_local[v__10.s7])) + ((convert_float8(vload8(0, layer_norm99 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 4096)))) * (convert_float8(vload8(0, dequantize_local + 0)))); NT_matmul_intermediate_pad_rf_local[v__10.s0] = v__11.s0; NT_matmul_intermediate_pad_rf_local[v__10.s1] = v__11.s1; NT_matmul_intermediate_pad_rf_local[v__10.s2] = v__11.s2; NT_matmul_intermediate_pad_rf_local[v__10.s3] = v__11.s3; NT_matmul_intermediate_pad_rf_local[v__10.s4] = v__11.s4; NT_matmul_intermediate_pad_rf_local[v__10.s5] = v__11.s5; NT_matmul_intermediate_pad_rf_local[v__10.s6] = v__11.s6; NT_matmul_intermediate_pad_rf_local[v__10.s7] = v__11.s7; int8 v__12 = (int8)((5)+(8*0), (5)+(8*1), (5)+(8*2), (5)+(8*3), (5)+(8*4), (5)+(8*5), (5)+(8*6), (5)+(8*7)); float8 v__13 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__12.s0],NT_matmul_intermediate_pad_rf_local[v__12.s1],NT_matmul_intermediate_pad_rf_local[v__12.s2],NT_matmul_intermediate_pad_rf_local[v__12.s3],NT_matmul_intermediate_pad_rf_local[v__12.s4],NT_matmul_intermediate_pad_rf_local[v__12.s5],NT_matmul_intermediate_pad_rf_local[v__12.s6],NT_matmul_intermediate_pad_rf_local[v__12.s7])) + ((convert_float8(vload8(0, layer_norm99 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 4096)))) * (convert_float8(vload8(0, dequantize_local + 8)))); NT_matmul_intermediate_pad_rf_local[v__12.s0] = v__13.s0; NT_matmul_intermediate_pad_rf_local[v__12.s1] = v__13.s1; NT_matmul_intermediate_pad_rf_local[v__12.s2] = v__13.s2; NT_matmul_intermediate_pad_rf_local[v__12.s3] = v__13.s3; NT_matmul_intermediate_pad_rf_local[v__12.s4] = v__13.s4; NT_matmul_intermediate_pad_rf_local[v__12.s5] = v__13.s5; NT_matmul_intermediate_pad_rf_local[v__12.s6] = v__13.s6; NT_matmul_intermediate_pad_rf_local[v__12.s7] = v__13.s7; } if ((((convert_int(get_group_id(1))) * 4) + 3) < seq_len) { int8 v__14 = (int8)((6)+(8*0), (6)+(8*1), (6)+(8*2), (6)+(8*3), (6)+(8*4), (6)+(8*5), (6)+(8*6), (6)+(8*7)); float8 v__15 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__14.s0],NT_matmul_intermediate_pad_rf_local[v__14.s1],NT_matmul_intermediate_pad_rf_local[v__14.s2],NT_matmul_intermediate_pad_rf_local[v__14.s3],NT_matmul_intermediate_pad_rf_local[v__14.s4],NT_matmul_intermediate_pad_rf_local[v__14.s5],NT_matmul_intermediate_pad_rf_local[v__14.s6],NT_matmul_intermediate_pad_rf_local[v__14.s7])) + ((convert_float8(vload8(0, layer_norm99 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 6144)))) * (convert_float8(vload8(0, dequantize_local + 0)))); NT_matmul_intermediate_pad_rf_local[v__14.s0] = v__15.s0; NT_matmul_intermediate_pad_rf_local[v__14.s1] = v__15.s1; NT_matmul_intermediate_pad_rf_local[v__14.s2] = v__15.s2; NT_matmul_intermediate_pad_rf_local[v__14.s3] = v__15.s3; NT_matmul_intermediate_pad_rf_local[v__14.s4] = v__15.s4; NT_matmul_intermediate_pad_rf_local[v__14.s5] = v__15.s5; NT_matmul_intermediate_pad_rf_local[v__14.s6] = v__15.s6; NT_matmul_intermediate_pad_rf_local[v__14.s7] = v__15.s7; int8 v__16 = (int8)((7)+(8*0), (7)+(8*1), (7)+(8*2), (7)+(8*3), (7)+(8*4), (7)+(8*5), (7)+(8*6), (7)+(8*7)); float8 v__17 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__16.s0],NT_matmul_intermediate_pad_rf_local[v__16.s1],NT_matmul_intermediate_pad_rf_local[v__16.s2],NT_matmul_intermediate_pad_rf_local[v__16.s3],NT_matmul_intermediate_pad_rf_local[v__16.s4],NT_matmul_intermediate_pad_rf_local[v__16.s5],NT_matmul_intermediate_pad_rf_local[v__16.s6],NT_matmul_intermediate_pad_rf_local[v__16.s7])) + ((convert_float8(vload8(0, layer_norm99 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 6144)))) * (convert_float8(vload8(0, dequantize_local + 8)))); NT_matmul_intermediate_pad_rf_local[v__16.s0] = v__17.s0; NT_matmul_intermediate_pad_rf_local[v__16.s1] = v__17.s1; NT_matmul_intermediate_pad_rf_local[v__16.s2] = v__17.s2; NT_matmul_intermediate_pad_rf_local[v__16.s3] = v__17.s3; NT_matmul_intermediate_pad_rf_local[v__16.s4] = v__17.s4; NT_matmul_intermediate_pad_rf_local[v__16.s5] = v__17.s5; NT_matmul_intermediate_pad_rf_local[v__16.s6] = v__17.s6; NT_matmul_intermediate_pad_rf_local[v__16.s7] = v__17.s7; } } for (int ax3_fused_2_0 = 0; ax3_fused_2_0 < 1; ++ax3_fused_2_0) { for (int ax2 = 0; ax2 < 4; ++ax2) { vstore2(((float2)(0.000000e+00f, 0.000000e+00f)), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + (ax2 * 2))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 8))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 16))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 24))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 32))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 40))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 48))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 56))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); } } for (int ax2_fused_2 = 0; ax2_fused_2 < 2; ++ax2_fused_2) { for (int ax1 = 0; ax1 < 4; ++ax1) { barrier(CLK_LOCAL_MEM_FENCE); ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_intermediate_pad_rf_local_1[((ax1 * 2) + ax2_fused_2)]; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 16) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 8) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 4) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 2) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); } barrier(CLK_LOCAL_MEM_FENCE); NT_matmul_intermediate_pad_local[((ax1 * 2) + ax2_fused_2)] = ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) * 32)]; } } for (int ax0 = 0; ax0 < 4; ++ax0) { for (int ax1_fused_2 = 0; ax1_fused_2 < 2; ++ax1_fused_2) { if (((bool)(((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)(((bool)(((convert_int(get_group_id(1))) - ((seq_len + 3) >> 2)) < 0)) || ((bool)((convert_int(get_group_id(1))) == 0)))))) && ((bool)((((convert_int(get_group_id(1))) * 4) + ax0) < seq_len))) { compute[((((((convert_int(get_group_id(1))) * 32768) + (ax0 * 8192)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] = (convert_half(((NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]) * 7.071068e-01f)) * 5.000000e-01f))))); } } } } // Function: gpt_neox_q4f16_1_tree_attn_paged_kv_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset) { int tile_id[1]; int batch_idx[1]; int batch_rows[1]; int batch_tiles[1]; int kv_chunk_len[1]; __local float m_smem[32]; __local float d_smem[32]; float O_local[32]; __local half Q_smem[4096]; __local half K_smem[4096]; __local half V_smem[4096]; float S_local[8]; float m_prev[1]; float m_new[1]; float d_new[1]; __local float m_prev_smem[32]; tile_id[0] = (convert_int(get_group_id(0))); batch_idx[0] = 0; batch_rows[0] = (q_indptr[(q_indptr_elem_offset + 1)] - q_indptr[q_indptr_elem_offset]); batch_tiles[0] = ((batch_rows[0] + 31) >> 5); while (1) { if (!(((batch_idx[0] < batch_size)))) { break; } while (1) { if (!((((bool)(batch_tiles[0] <= tile_id[0])) && ((bool)(batch_idx[0] < batch_size))))) { break; } tile_id[0] = (tile_id[0] - batch_tiles[0]); batch_idx[0] = (batch_idx[0] + 1); if (batch_idx[0] < batch_size) { int b_idx = batch_idx[0]; batch_rows[0] = (q_indptr[((b_idx + q_indptr_elem_offset) + 1)] - q_indptr[(b_idx + q_indptr_elem_offset)]); batch_tiles[0] = ((batch_rows[0] + 31) >> 5); } } if ((batch_idx[0] < batch_size)) { int b_idx_1 = batch_idx[0]; int LH_start = (tile_id[0] * 32); int q_indptr_val = q_indptr[(b_idx_1 + q_indptr_elem_offset)]; int cur_page_indptr_begin = page_indptr[(b_idx_1 + page_indptr_elem_offset)]; int cur_page_indptr_end = page_indptr[((b_idx_1 + page_indptr_elem_offset) + 1)]; int condval; if ((cur_page_indptr_begin != cur_page_indptr_end)) { condval = ((((cur_page_indptr_end * 32) + length_info[(b_idx_1 + length_info_elem_offset)]) - (cur_page_indptr_begin * 32)) - 32); } else { condval = 0; } kv_chunk_len[0] = condval; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { m_smem[(convert_int(get_local_id(0)))] = -5.000000e+04f; d_smem[(convert_int(get_local_id(0)))] = 1.000000e+00f; } for (int li_1 = 0; li_1 < 4; ++li_1) { for (int lj_1 = 0; lj_1 < 8; ++lj_1) { O_local[((li_1 * 8) + lj_1)] = 0.000000e+00f; } } barrier(CLK_LOCAL_MEM_FENCE); for (int li_lj_fused_0 = 0; li_lj_fused_0 < 8; ++li_lj_fused_0) { if (((((li_lj_fused_0 * 4) + q_indptr_val) + LH_start) + (convert_int(get_local_id(1)))) < q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)]) { vstore4(vload4(0, q + ((((((li_lj_fused_0 * 8192) + ((convert_int(get_local_id(1))) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))), 0, Q_smem + (((li_lj_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); } else { vstore4(((half4)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, Q_smem + (((li_lj_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); } } barrier(CLK_LOCAL_MEM_FENCE); for (int iterator = 0; iterator < ((kv_chunk_len[0] + 31) >> 5); ++iterator) { barrier(CLK_LOCAL_MEM_FENCE); for (int lz_ly_fused_0 = 0; lz_ly_fused_0 < 8; ++lz_ly_fused_0) { if ((((iterator * 32) + (lz_ly_fused_0 * 4)) + (convert_int(get_local_id(1)))) < kv_chunk_len[0]) { int page_no = page_values[((iterator + cur_page_indptr_begin) + page_values_elem_offset)]; vstore4(vload4(0, pages + (((((page_no * 131072) + ((convert_int(get_group_id(1))) * 4096)) + (lz_ly_fused_0 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))), 0, K_smem + (((lz_ly_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); } else { vstore4(((half4)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, K_smem + (((lz_ly_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); } } barrier(CLK_LOCAL_MEM_FENCE); for (int lz_ly_fused_0_1 = 0; lz_ly_fused_0_1 < 8; ++lz_ly_fused_0_1) { if ((((iterator * 32) + (lz_ly_fused_0_1 * 4)) + (convert_int(get_local_id(1)))) < kv_chunk_len[0]) { int page_no_1 = page_values[((iterator + cur_page_indptr_begin) + page_values_elem_offset)]; vstore4(vload4(0, pages + ((((((page_no_1 * 131072) + ((convert_int(get_group_id(1))) * 4096)) + (lz_ly_fused_0_1 * 512)) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + 65536)), 0, V_smem + (((lz_ly_fused_0_1 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); } else { vstore4(((half4)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, V_smem + (((lz_ly_fused_0_1 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); } } barrier(CLK_LOCAL_MEM_FENCE); for (int li_1_init = 0; li_1_init < 2; ++li_1_init) { for (int lj_1_init = 0; lj_1_init < 4; ++lj_1_init) { S_local[((li_1_init * 4) + lj_1_init)] = 0.000000e+00f; } } for (int lk_0 = 0; lk_0 < 16; ++lk_0) { for (int li_1_1 = 0; li_1_1 < 2; ++li_1_1) { for (int lj_1_1 = 0; lj_1_1 < 4; ++lj_1_1) { for (int lk_1 = 0; lk_1 < 8; ++lk_1) { S_local[((li_1_1 * 4) + lj_1_1)] = (S_local[((li_1_1 * 4) + lj_1_1)] + ((((convert_float(Q_smem[((((((convert_int(get_local_id(1))) * 1024) + (((convert_int(get_local_id(0))) >> 3) * 256)) + (li_1_1 * 128)) + (lk_0 * 8)) + lk_1)])) * (convert_float(K_smem[((((((convert_int(get_local_id(0))) & 7) * 512) + (lj_1_1 * 128)) + (lk_0 * 8)) + lk_1)]))) * attn_score_scaling_factor) * 1.275174e-01f)); } } } } barrier(CLK_LOCAL_MEM_FENCE); for (int li_1_2 = 0; li_1_2 < 2; ++li_1_2) { for (int lj_1_2 = 0; lj_1_2 < 4; ++lj_1_2) { ((__local float*)K_smem)[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 3) * 64)) + (li_1_2 * 32)) + (((convert_int(get_local_id(0))) & 7) * 4)) + lj_1_2)] = S_local[((li_1_2 * 4) + lj_1_2)]; } } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { m_prev[0] = m_smem[(convert_int(get_local_id(0)))]; m_new[0] = m_smem[(convert_int(get_local_id(0)))]; for (int j = 0; j < 32; ++j) { int cse_var_6 = ((iterator * 32) + j); int cse_var_5 = ((iterator * 64) + (j * 2)); if (((bool)(cse_var_6 < kv_chunk_len[0])) && ((bool)(((bool)(cse_var_6 < ((kv_chunk_len[0] + tree_order_indptr[(b_idx_1 + tree_order_indptr_elem_offset)]) - tree_order_indptr[((b_idx_1 + tree_order_indptr_elem_offset) + 1)]))) || ((bool)(((bool)(tree_order[((((tree_order_indptr[((b_idx_1 + tree_order_indptr_elem_offset) + 1)] * 2) + cse_var_5) + tree_order_elem_offset) - (kv_chunk_len[0] * 2))] <= tree_order[(((((((convert_int(get_local_id(0))) * 2) + (LH_start * 2)) + (tree_order_indptr[((b_idx_1 + tree_order_indptr_elem_offset) + 1)] * 2)) + (q_indptr[(b_idx_1 + q_indptr_elem_offset)] * 2)) + tree_order_elem_offset) - (q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)] * 2))])) && ((bool)(tree_order[(((((((convert_int(get_local_id(0))) * 2) + (LH_start * 2)) + (tree_order_indptr[((b_idx_1 + tree_order_indptr_elem_offset) + 1)] * 2)) + (q_indptr[(b_idx_1 + q_indptr_elem_offset)] * 2)) + tree_order_elem_offset) - (q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)] * 2))] < tree_order[(((((tree_order_indptr[((b_idx_1 + tree_order_indptr_elem_offset) + 1)] * 2) + cse_var_5) + tree_order_elem_offset) + 1) - (kv_chunk_len[0] * 2))]))))))) { m_new[0] = max((float)m_new[0], (float)((__local float*)K_smem)[((((convert_int(get_local_id(1))) * 1024) + ((convert_int(get_local_id(0))) * 32)) + j)]); } } d_new[0] = (d_smem[(convert_int(get_local_id(0)))] * exp2((m_prev[0] - m_new[0]))); } for (int j_1 = 0; j_1 < 32; ++j_1) { barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { int cse_var_8 = ((iterator * 32) + j_1); int cse_var_7 = ((iterator * 64) + (j_1 * 2)); if (((bool)(cse_var_8 < kv_chunk_len[0])) && ((bool)(((bool)(cse_var_8 < ((kv_chunk_len[0] + tree_order_indptr[(b_idx_1 + tree_order_indptr_elem_offset)]) - tree_order_indptr[((b_idx_1 + tree_order_indptr_elem_offset) + 1)]))) || ((bool)(((bool)(tree_order[((((tree_order_indptr[((b_idx_1 + tree_order_indptr_elem_offset) + 1)] * 2) + cse_var_7) + tree_order_elem_offset) - (kv_chunk_len[0] * 2))] <= tree_order[(((((((convert_int(get_local_id(0))) * 2) + (LH_start * 2)) + (tree_order_indptr[((b_idx_1 + tree_order_indptr_elem_offset) + 1)] * 2)) + (q_indptr[(b_idx_1 + q_indptr_elem_offset)] * 2)) + tree_order_elem_offset) - (q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)] * 2))])) && ((bool)(tree_order[(((((((convert_int(get_local_id(0))) * 2) + (LH_start * 2)) + (tree_order_indptr[((b_idx_1 + tree_order_indptr_elem_offset) + 1)] * 2)) + (q_indptr[(b_idx_1 + q_indptr_elem_offset)] * 2)) + tree_order_elem_offset) - (q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)] * 2))] < tree_order[(((((tree_order_indptr[((b_idx_1 + tree_order_indptr_elem_offset) + 1)] * 2) + cse_var_7) + tree_order_elem_offset) + 1) - (kv_chunk_len[0] * 2))]))))))) { ((__local float*)K_smem)[((((convert_int(get_local_id(1))) * 1024) + ((convert_int(get_local_id(0))) * 32)) + j_1)] = exp2((((__local float*)K_smem)[((((convert_int(get_local_id(1))) * 1024) + ((convert_int(get_local_id(0))) * 32)) + j_1)] - m_new[0])); } else { ((__local float*)K_smem)[(((convert_int(get_local_id(0))) * 32) + j_1)] = exp2((-5.000000e+04f - m_new[0])); } } } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { for (int j_2 = 0; j_2 < 32; ++j_2) { d_new[0] = (d_new[0] + ((__local float*)K_smem)[(((convert_int(get_local_id(0))) * 32) + j_2)]); } m_smem[(convert_int(get_local_id(0)))] = m_new[0]; d_smem[(convert_int(get_local_id(0)))] = d_new[0]; m_prev_smem[(convert_int(get_local_id(0)))] = m_prev[0]; } barrier(CLK_LOCAL_MEM_FENCE); for (int li_1_init_1 = 0; li_1_init_1 < 4; ++li_1_init_1) { for (int lj_1_init_1 = 0; lj_1_init_1 < 8; ++lj_1_init_1) { O_local[((li_1_init_1 * 8) + lj_1_init_1)] = (O_local[((li_1_init_1 * 8) + lj_1_init_1)] * exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_init_1)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_init_1)]))); } } for (int lk_0_1 = 0; lk_0_1 < 4; ++lk_0_1) { for (int lk_1_1 = 0; lk_1_1 < 8; ++lk_1_1) { for (int li_1_3 = 0; li_1_3 < 4; ++li_1_3) { for (int lj_1_3 = 0; lj_1_3 < 8; ++lj_1_3) { O_local[((li_1_3 * 8) + lj_1_3)] = (O_local[((li_1_3 * 8) + lj_1_3)] + (((__local float*)K_smem)[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (li_1_3 * 32)) + (lk_0_1 * 8)) + lk_1_1)] * (convert_float(V_smem[((((lk_0_1 * 1024) + (lk_1_1 * 128)) + (((convert_int(get_local_id(0))) & 15) * 8)) + lj_1_3)])))); } } } } } for (int li_1_4 = 0; li_1_4 < 4; ++li_1_4) { for (int lj_1_4 = 0; lj_1_4 < 8; ++lj_1_4) { int cur_L = ((((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + q_indptr[(b_idx_1 + q_indptr_elem_offset)]) + LH_start) + li_1_4); if (cur_L < q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)]) { output[((((cur_L * 2048) + ((convert_int(get_group_id(1))) * 128)) + (((convert_int(get_local_id(0))) & 15) * 8)) + lj_1_4)] = (convert_half((O_local[((li_1_4 * 8) + lj_1_4)] / d_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_4)]))); } } } if ((convert_int(get_local_id(1))) < 1) { int cur_L_1 = (((((convert_int(get_local_id(1))) * 32) + q_indptr[(b_idx_1 + q_indptr_elem_offset)]) + LH_start) + (convert_int(get_local_id(0)))); if (cur_L_1 < q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)]) { lse[((cur_L_1 * 16) + (convert_int(get_group_id(1))))] = (m_smem[(convert_int(get_local_id(0)))] + log2(d_smem[(convert_int(get_local_id(0)))])); } } tile_id[0] = (tile_id[0] + 16); } } } // Function: gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size) { half NT_matmul_rf_local[8]; uint embed_out_q_weight2_local[1]; half NT_matmul_rf_local_1[1]; half in_thread_NT_matmul_intermediate_local[1]; __local half red_buf0[64]; if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { vstore8(((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, NT_matmul_rf_local + 0); } for (int ax1_0_fused_ax1_1_fused_0 = 0; ax1_0_fused_ax1_1_fused_0 < 8; ++ax1_0_fused_ax1_1_fused_0) { if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { embed_out_q_weight2_local[0] = embed_out_q_weight2[(((((convert_int(get_group_id(0))) * 512) + ((convert_int(get_local_id(0))) * 256)) + (ax1_0_fused_ax1_1_fused_0 * 32)) + (convert_int(get_local_id(1))))]; } if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { vstore8((vload8(0, NT_matmul_rf_local + 0) + (vload8(0, layer_norm97 + ((ax1_0_fused_ax1_1_fused_0 * 256) + ((convert_int(get_local_id(1))) * 8))) * (((convert_half8(((((uint8)(embed_out_q_weight2_local[0], embed_out_q_weight2_local[0], embed_out_q_weight2_local[0], embed_out_q_weight2_local[0], embed_out_q_weight2_local[0], embed_out_q_weight2_local[0], embed_out_q_weight2_local[0], embed_out_q_weight2_local[0])) >> ((uint8)(((uint)0)+((uint)4*0), ((uint)0)+((uint)4*1), ((uint)0)+((uint)4*2), ((uint)0)+((uint)4*3), ((uint)0)+((uint)4*4), ((uint)0)+((uint)4*5), ((uint)0)+((uint)4*6), ((uint)0)+((uint)4*7)))) & ((uint8)((uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15))))) - ((half8)((half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f))) * ((half8)(embed_out_q_scale2[(((((convert_int(get_group_id(0))) * 128) + ((convert_int(get_local_id(0))) * 64)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], embed_out_q_scale2[(((((convert_int(get_group_id(0))) * 128) + ((convert_int(get_local_id(0))) * 64)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], embed_out_q_scale2[(((((convert_int(get_group_id(0))) * 128) + ((convert_int(get_local_id(0))) * 64)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], embed_out_q_scale2[(((((convert_int(get_group_id(0))) * 128) + ((convert_int(get_local_id(0))) * 64)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], embed_out_q_scale2[(((((convert_int(get_group_id(0))) * 128) + ((convert_int(get_local_id(0))) * 64)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], embed_out_q_scale2[(((((convert_int(get_group_id(0))) * 128) + ((convert_int(get_local_id(0))) * 64)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], embed_out_q_scale2[(((((convert_int(get_group_id(0))) * 128) + ((convert_int(get_local_id(0))) * 64)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], embed_out_q_scale2[(((((convert_int(get_group_id(0))) * 128) + ((convert_int(get_local_id(0))) * 64)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]))))), 0, NT_matmul_rf_local + 0); } } for (int ax2_fused_2_0 = 0; ax2_fused_2_0 < 1; ++ax2_fused_2_0) { if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { NT_matmul_rf_local_1[0] = (half)0.000000e+00f; } if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[0]); } if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[1]); } if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[2]); } if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[3]); } if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[4]); } if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[5]); } if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[6]); } if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[7]); } } in_thread_NT_matmul_intermediate_local[0] = (half)0.000000e+00f; if ((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size) { in_thread_NT_matmul_intermediate_local[0] = (in_thread_NT_matmul_intermediate_local[0] + NT_matmul_rf_local_1[0]); } barrier(CLK_LOCAL_MEM_FENCE); ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = in_thread_NT_matmul_intermediate_local[0]; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 16) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 8) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 4) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 2) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); } barrier(CLK_LOCAL_MEM_FENCE); if (((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)((((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0)))) < vocab_size))) { compute[(((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0))))] = (convert_float(((volatile __local half*)red_buf0)[((convert_int(get_local_id(0))) * 32)])); } } // Function: gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1 #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size) { float NT_matmul_intermediate_pad_rf_local[64]; half dequantize_local[16]; float NT_matmul_intermediate_pad_rf_local_1[8]; float NT_matmul_intermediate_pad_local[8]; __local float red_buf0[64]; for (int ax0_1_init = 0; ax0_1_init < 4; ++ax0_1_init) { for (int ax1_fused_2_init = 0; ax1_fused_2_init < 2; ++ax1_fused_2_init) { int8 v_ = (int8)((((ax0_1_init * 2) + ax1_fused_2_init))+(8*0), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*1), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*2), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*3), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*4), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*5), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*6), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*7)); float8 v__1 = (float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f); NT_matmul_intermediate_pad_rf_local[v_.s0] = v__1.s0; NT_matmul_intermediate_pad_rf_local[v_.s1] = v__1.s1; NT_matmul_intermediate_pad_rf_local[v_.s2] = v__1.s2; NT_matmul_intermediate_pad_rf_local[v_.s3] = v__1.s3; NT_matmul_intermediate_pad_rf_local[v_.s4] = v__1.s4; NT_matmul_intermediate_pad_rf_local[v_.s5] = v__1.s5; NT_matmul_intermediate_pad_rf_local[v_.s6] = v__1.s6; NT_matmul_intermediate_pad_rf_local[v_.s7] = v__1.s7; } } for (int ax2_fused_0 = 0; ax2_fused_0 < 8; ++ax2_fused_0) { for (int ax0_0 = 0; ax0_0 < 2; ++ax0_0) { dequantize_local[(ax0_0 * 8)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)0) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 1)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)4) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 2)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)8) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 3)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)12) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 4)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)16) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 5)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)20) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 6)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)24) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 7)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)28) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); } if (((convert_int(get_group_id(1))) * 4) < batch_size) { int8 v__2 = (int8)((0)+(8*0), (0)+(8*1), (0)+(8*2), (0)+(8*3), (0)+(8*4), (0)+(8*5), (0)+(8*6), (0)+(8*7)); float8 v__3 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__2.s0],NT_matmul_intermediate_pad_rf_local[v__2.s1],NT_matmul_intermediate_pad_rf_local[v__2.s2],NT_matmul_intermediate_pad_rf_local[v__2.s3],NT_matmul_intermediate_pad_rf_local[v__2.s4],NT_matmul_intermediate_pad_rf_local[v__2.s5],NT_matmul_intermediate_pad_rf_local[v__2.s6],NT_matmul_intermediate_pad_rf_local[v__2.s7])) + ((convert_float8(vload8(0, layer_norm148 + ((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, dequantize_local + 0)))); NT_matmul_intermediate_pad_rf_local[v__2.s0] = v__3.s0; NT_matmul_intermediate_pad_rf_local[v__2.s1] = v__3.s1; NT_matmul_intermediate_pad_rf_local[v__2.s2] = v__3.s2; NT_matmul_intermediate_pad_rf_local[v__2.s3] = v__3.s3; NT_matmul_intermediate_pad_rf_local[v__2.s4] = v__3.s4; NT_matmul_intermediate_pad_rf_local[v__2.s5] = v__3.s5; NT_matmul_intermediate_pad_rf_local[v__2.s6] = v__3.s6; NT_matmul_intermediate_pad_rf_local[v__2.s7] = v__3.s7; int8 v__4 = (int8)((1)+(8*0), (1)+(8*1), (1)+(8*2), (1)+(8*3), (1)+(8*4), (1)+(8*5), (1)+(8*6), (1)+(8*7)); float8 v__5 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__4.s0],NT_matmul_intermediate_pad_rf_local[v__4.s1],NT_matmul_intermediate_pad_rf_local[v__4.s2],NT_matmul_intermediate_pad_rf_local[v__4.s3],NT_matmul_intermediate_pad_rf_local[v__4.s4],NT_matmul_intermediate_pad_rf_local[v__4.s5],NT_matmul_intermediate_pad_rf_local[v__4.s6],NT_matmul_intermediate_pad_rf_local[v__4.s7])) + ((convert_float8(vload8(0, layer_norm148 + ((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, dequantize_local + 8)))); NT_matmul_intermediate_pad_rf_local[v__4.s0] = v__5.s0; NT_matmul_intermediate_pad_rf_local[v__4.s1] = v__5.s1; NT_matmul_intermediate_pad_rf_local[v__4.s2] = v__5.s2; NT_matmul_intermediate_pad_rf_local[v__4.s3] = v__5.s3; NT_matmul_intermediate_pad_rf_local[v__4.s4] = v__5.s4; NT_matmul_intermediate_pad_rf_local[v__4.s5] = v__5.s5; NT_matmul_intermediate_pad_rf_local[v__4.s6] = v__5.s6; NT_matmul_intermediate_pad_rf_local[v__4.s7] = v__5.s7; } if ((((convert_int(get_group_id(1))) * 4) + 1) < batch_size) { int8 v__6 = (int8)((2)+(8*0), (2)+(8*1), (2)+(8*2), (2)+(8*3), (2)+(8*4), (2)+(8*5), (2)+(8*6), (2)+(8*7)); float8 v__7 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__6.s0],NT_matmul_intermediate_pad_rf_local[v__6.s1],NT_matmul_intermediate_pad_rf_local[v__6.s2],NT_matmul_intermediate_pad_rf_local[v__6.s3],NT_matmul_intermediate_pad_rf_local[v__6.s4],NT_matmul_intermediate_pad_rf_local[v__6.s5],NT_matmul_intermediate_pad_rf_local[v__6.s6],NT_matmul_intermediate_pad_rf_local[v__6.s7])) + ((convert_float8(vload8(0, layer_norm148 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)))) * (convert_float8(vload8(0, dequantize_local + 0)))); NT_matmul_intermediate_pad_rf_local[v__6.s0] = v__7.s0; NT_matmul_intermediate_pad_rf_local[v__6.s1] = v__7.s1; NT_matmul_intermediate_pad_rf_local[v__6.s2] = v__7.s2; NT_matmul_intermediate_pad_rf_local[v__6.s3] = v__7.s3; NT_matmul_intermediate_pad_rf_local[v__6.s4] = v__7.s4; NT_matmul_intermediate_pad_rf_local[v__6.s5] = v__7.s5; NT_matmul_intermediate_pad_rf_local[v__6.s6] = v__7.s6; NT_matmul_intermediate_pad_rf_local[v__6.s7] = v__7.s7; int8 v__8 = (int8)((3)+(8*0), (3)+(8*1), (3)+(8*2), (3)+(8*3), (3)+(8*4), (3)+(8*5), (3)+(8*6), (3)+(8*7)); float8 v__9 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__8.s0],NT_matmul_intermediate_pad_rf_local[v__8.s1],NT_matmul_intermediate_pad_rf_local[v__8.s2],NT_matmul_intermediate_pad_rf_local[v__8.s3],NT_matmul_intermediate_pad_rf_local[v__8.s4],NT_matmul_intermediate_pad_rf_local[v__8.s5],NT_matmul_intermediate_pad_rf_local[v__8.s6],NT_matmul_intermediate_pad_rf_local[v__8.s7])) + ((convert_float8(vload8(0, layer_norm148 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)))) * (convert_float8(vload8(0, dequantize_local + 8)))); NT_matmul_intermediate_pad_rf_local[v__8.s0] = v__9.s0; NT_matmul_intermediate_pad_rf_local[v__8.s1] = v__9.s1; NT_matmul_intermediate_pad_rf_local[v__8.s2] = v__9.s2; NT_matmul_intermediate_pad_rf_local[v__8.s3] = v__9.s3; NT_matmul_intermediate_pad_rf_local[v__8.s4] = v__9.s4; NT_matmul_intermediate_pad_rf_local[v__8.s5] = v__9.s5; NT_matmul_intermediate_pad_rf_local[v__8.s6] = v__9.s6; NT_matmul_intermediate_pad_rf_local[v__8.s7] = v__9.s7; } if ((((convert_int(get_group_id(1))) * 4) + 2) < batch_size) { int8 v__10 = (int8)((4)+(8*0), (4)+(8*1), (4)+(8*2), (4)+(8*3), (4)+(8*4), (4)+(8*5), (4)+(8*6), (4)+(8*7)); float8 v__11 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__10.s0],NT_matmul_intermediate_pad_rf_local[v__10.s1],NT_matmul_intermediate_pad_rf_local[v__10.s2],NT_matmul_intermediate_pad_rf_local[v__10.s3],NT_matmul_intermediate_pad_rf_local[v__10.s4],NT_matmul_intermediate_pad_rf_local[v__10.s5],NT_matmul_intermediate_pad_rf_local[v__10.s6],NT_matmul_intermediate_pad_rf_local[v__10.s7])) + ((convert_float8(vload8(0, layer_norm148 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 4096)))) * (convert_float8(vload8(0, dequantize_local + 0)))); NT_matmul_intermediate_pad_rf_local[v__10.s0] = v__11.s0; NT_matmul_intermediate_pad_rf_local[v__10.s1] = v__11.s1; NT_matmul_intermediate_pad_rf_local[v__10.s2] = v__11.s2; NT_matmul_intermediate_pad_rf_local[v__10.s3] = v__11.s3; NT_matmul_intermediate_pad_rf_local[v__10.s4] = v__11.s4; NT_matmul_intermediate_pad_rf_local[v__10.s5] = v__11.s5; NT_matmul_intermediate_pad_rf_local[v__10.s6] = v__11.s6; NT_matmul_intermediate_pad_rf_local[v__10.s7] = v__11.s7; int8 v__12 = (int8)((5)+(8*0), (5)+(8*1), (5)+(8*2), (5)+(8*3), (5)+(8*4), (5)+(8*5), (5)+(8*6), (5)+(8*7)); float8 v__13 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__12.s0],NT_matmul_intermediate_pad_rf_local[v__12.s1],NT_matmul_intermediate_pad_rf_local[v__12.s2],NT_matmul_intermediate_pad_rf_local[v__12.s3],NT_matmul_intermediate_pad_rf_local[v__12.s4],NT_matmul_intermediate_pad_rf_local[v__12.s5],NT_matmul_intermediate_pad_rf_local[v__12.s6],NT_matmul_intermediate_pad_rf_local[v__12.s7])) + ((convert_float8(vload8(0, layer_norm148 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 4096)))) * (convert_float8(vload8(0, dequantize_local + 8)))); NT_matmul_intermediate_pad_rf_local[v__12.s0] = v__13.s0; NT_matmul_intermediate_pad_rf_local[v__12.s1] = v__13.s1; NT_matmul_intermediate_pad_rf_local[v__12.s2] = v__13.s2; NT_matmul_intermediate_pad_rf_local[v__12.s3] = v__13.s3; NT_matmul_intermediate_pad_rf_local[v__12.s4] = v__13.s4; NT_matmul_intermediate_pad_rf_local[v__12.s5] = v__13.s5; NT_matmul_intermediate_pad_rf_local[v__12.s6] = v__13.s6; NT_matmul_intermediate_pad_rf_local[v__12.s7] = v__13.s7; } if ((((convert_int(get_group_id(1))) * 4) + 3) < batch_size) { int8 v__14 = (int8)((6)+(8*0), (6)+(8*1), (6)+(8*2), (6)+(8*3), (6)+(8*4), (6)+(8*5), (6)+(8*6), (6)+(8*7)); float8 v__15 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__14.s0],NT_matmul_intermediate_pad_rf_local[v__14.s1],NT_matmul_intermediate_pad_rf_local[v__14.s2],NT_matmul_intermediate_pad_rf_local[v__14.s3],NT_matmul_intermediate_pad_rf_local[v__14.s4],NT_matmul_intermediate_pad_rf_local[v__14.s5],NT_matmul_intermediate_pad_rf_local[v__14.s6],NT_matmul_intermediate_pad_rf_local[v__14.s7])) + ((convert_float8(vload8(0, layer_norm148 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 6144)))) * (convert_float8(vload8(0, dequantize_local + 0)))); NT_matmul_intermediate_pad_rf_local[v__14.s0] = v__15.s0; NT_matmul_intermediate_pad_rf_local[v__14.s1] = v__15.s1; NT_matmul_intermediate_pad_rf_local[v__14.s2] = v__15.s2; NT_matmul_intermediate_pad_rf_local[v__14.s3] = v__15.s3; NT_matmul_intermediate_pad_rf_local[v__14.s4] = v__15.s4; NT_matmul_intermediate_pad_rf_local[v__14.s5] = v__15.s5; NT_matmul_intermediate_pad_rf_local[v__14.s6] = v__15.s6; NT_matmul_intermediate_pad_rf_local[v__14.s7] = v__15.s7; int8 v__16 = (int8)((7)+(8*0), (7)+(8*1), (7)+(8*2), (7)+(8*3), (7)+(8*4), (7)+(8*5), (7)+(8*6), (7)+(8*7)); float8 v__17 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__16.s0],NT_matmul_intermediate_pad_rf_local[v__16.s1],NT_matmul_intermediate_pad_rf_local[v__16.s2],NT_matmul_intermediate_pad_rf_local[v__16.s3],NT_matmul_intermediate_pad_rf_local[v__16.s4],NT_matmul_intermediate_pad_rf_local[v__16.s5],NT_matmul_intermediate_pad_rf_local[v__16.s6],NT_matmul_intermediate_pad_rf_local[v__16.s7])) + ((convert_float8(vload8(0, layer_norm148 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 6144)))) * (convert_float8(vload8(0, dequantize_local + 8)))); NT_matmul_intermediate_pad_rf_local[v__16.s0] = v__17.s0; NT_matmul_intermediate_pad_rf_local[v__16.s1] = v__17.s1; NT_matmul_intermediate_pad_rf_local[v__16.s2] = v__17.s2; NT_matmul_intermediate_pad_rf_local[v__16.s3] = v__17.s3; NT_matmul_intermediate_pad_rf_local[v__16.s4] = v__17.s4; NT_matmul_intermediate_pad_rf_local[v__16.s5] = v__17.s5; NT_matmul_intermediate_pad_rf_local[v__16.s6] = v__17.s6; NT_matmul_intermediate_pad_rf_local[v__16.s7] = v__17.s7; } } for (int ax3_fused_2_0 = 0; ax3_fused_2_0 < 1; ++ax3_fused_2_0) { for (int ax2 = 0; ax2 < 4; ++ax2) { vstore2(((float2)(0.000000e+00f, 0.000000e+00f)), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + (ax2 * 2))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 8))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 16))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 24))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 32))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 40))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 48))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 56))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); } } for (int ax2_fused_2 = 0; ax2_fused_2 < 2; ++ax2_fused_2) { for (int ax1 = 0; ax1 < 4; ++ax1) { barrier(CLK_LOCAL_MEM_FENCE); ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_intermediate_pad_rf_local_1[((ax1 * 2) + ax2_fused_2)]; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 16) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 8) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 4) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 2) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); } barrier(CLK_LOCAL_MEM_FENCE); NT_matmul_intermediate_pad_local[((ax1 * 2) + ax2_fused_2)] = ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) * 32)]; } } for (int ax0 = 0; ax0 < 4; ++ax0) { for (int ax1_fused_2 = 0; ax1_fused_2 < 2; ++ax1_fused_2) { if (((bool)(((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)(((bool)(((convert_int(get_group_id(1))) - ((batch_size + 3) >> 2)) < 0)) || ((bool)((convert_int(get_group_id(1))) == 0)))))) && ((bool)((((convert_int(get_group_id(1))) * 4) + ax0) < batch_size))) { compute[((((((convert_int(get_group_id(1))) * 32768) + (ax0 * 8192)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] = (convert_half(((NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias4[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]) * 7.071068e-01f)) * 5.000000e-01f))))); } } } } // Function: gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2 #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size) { float8 NT_matmul_intermediate_reindex_pad_local[2]; for (int var = 0; var < 1; ++var) { NT_matmul_intermediate_reindex_pad_local[0] = ((float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)); NT_matmul_intermediate_reindex_pad_local[1] = ((float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)); for (int ax3_0 = 0; ax3_0 < 512; ++ax3_0) { for (int ax3_1 = 0; ax3_1 < 16; ++ax3_1) { uint cse_var_7 = (convert_uint(((ax3_1 & 7) * 4))); half condval; if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { condval = lv1[(((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + (ax3_0 * 16)) + ax3_1)]; } else { condval = (half)0.000000e+00f; } int8 v_ = (int8)(((((((convert_int(get_group_id(0))) * 262144) + ((convert_int(get_local_id(0))) * 8192)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(1024*0), ((((((convert_int(get_group_id(0))) * 262144) + ((convert_int(get_local_id(0))) * 8192)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(1024*1), ((((((convert_int(get_group_id(0))) * 262144) + ((convert_int(get_local_id(0))) * 8192)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(1024*2), ((((((convert_int(get_group_id(0))) * 262144) + ((convert_int(get_local_id(0))) * 8192)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(1024*3), ((((((convert_int(get_group_id(0))) * 262144) + ((convert_int(get_local_id(0))) * 8192)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(1024*4), ((((((convert_int(get_group_id(0))) * 262144) + ((convert_int(get_local_id(0))) * 8192)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(1024*5), ((((((convert_int(get_group_id(0))) * 262144) + ((convert_int(get_local_id(0))) * 8192)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(1024*6), ((((((convert_int(get_group_id(0))) * 262144) + ((convert_int(get_local_id(0))) * 8192)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(1024*7)); int8 v__1 = (int8)((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 >> 1)))+(256*0), (((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 >> 1)))+(256*1), (((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 >> 1)))+(256*2), (((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 >> 1)))+(256*3), (((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 >> 1)))+(256*4), (((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 >> 1)))+(256*5), (((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 >> 1)))+(256*6), (((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 >> 1)))+(256*7)); NT_matmul_intermediate_reindex_pad_local[0] = (((convert_float8(((half8)(condval, condval, condval, condval, condval, condval, condval, condval)))) * (convert_float8((((convert_half8(((((uint8)(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[v_.s0],gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[v_.s1],gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[v_.s2],gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[v_.s3],gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[v_.s4],gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[v_.s5],gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[v_.s6],gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[v_.s7])) >> ((uint8)(cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7))) & ((uint8)((uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15))))) - ((half8)((half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f))) * ((half8)(gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[v__1.s0],gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[v__1.s1],gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[v__1.s2],gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[v__1.s3],gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[v__1.s4],gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[v__1.s5],gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[v__1.s6],gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[v__1.s7])))))) + NT_matmul_intermediate_reindex_pad_local[0]); half condval_1; if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { condval_1 = lv1[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + (ax3_0 * 16)) + ax3_1) + 8192)]; } else { condval_1 = (half)0.000000e+00f; } int8 v__2 = (int8)(((((((convert_int(get_group_id(0))) * 262144) + ((convert_int(get_local_id(0))) * 8192)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(1024*0), ((((((convert_int(get_group_id(0))) * 262144) + ((convert_int(get_local_id(0))) * 8192)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(1024*1), ((((((convert_int(get_group_id(0))) * 262144) + ((convert_int(get_local_id(0))) * 8192)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(1024*2), ((((((convert_int(get_group_id(0))) * 262144) + ((convert_int(get_local_id(0))) * 8192)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(1024*3), ((((((convert_int(get_group_id(0))) * 262144) + ((convert_int(get_local_id(0))) * 8192)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(1024*4), ((((((convert_int(get_group_id(0))) * 262144) + ((convert_int(get_local_id(0))) * 8192)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(1024*5), ((((((convert_int(get_group_id(0))) * 262144) + ((convert_int(get_local_id(0))) * 8192)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(1024*6), ((((((convert_int(get_group_id(0))) * 262144) + ((convert_int(get_local_id(0))) * 8192)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(1024*7)); int8 v__3 = (int8)((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 >> 1)))+(256*0), (((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 >> 1)))+(256*1), (((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 >> 1)))+(256*2), (((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 >> 1)))+(256*3), (((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 >> 1)))+(256*4), (((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 >> 1)))+(256*5), (((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 >> 1)))+(256*6), (((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 >> 1)))+(256*7)); NT_matmul_intermediate_reindex_pad_local[1] = (((convert_float8(((half8)(condval_1, condval_1, condval_1, condval_1, condval_1, condval_1, condval_1, condval_1)))) * (convert_float8((((convert_half8(((((uint8)(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[v__2.s0],gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[v__2.s1],gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[v__2.s2],gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[v__2.s3],gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[v__2.s4],gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[v__2.s5],gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[v__2.s6],gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4[v__2.s7])) >> ((uint8)(cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7))) & ((uint8)((uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15))))) - ((half8)((half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f))) * ((half8)(gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[v__3.s0],gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[v__3.s1],gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[v__3.s2],gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[v__3.s3],gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[v__3.s4],gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[v__3.s5],gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[v__3.s6],gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4[v__3.s7])))))) + NT_matmul_intermediate_reindex_pad_local[1]); } } if ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size) { vstore8((convert_half8((NT_matmul_intermediate_reindex_pad_local[0] + vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_bias4 + (((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)))))), 0, compute + (((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8))); } if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size) { vstore8((convert_half8((NT_matmul_intermediate_reindex_pad_local[1] + vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_bias4 + (((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)))))), 0, compute + ((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 2048)); } } } // Function: gpt_neox_q4f16_1_apply_penalty_inplace_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size) { if ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) < num_token) { logits[((seq_ids[pos2seq_id[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]] * vocab_size) + token_ids[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))])] = (logits[((seq_ids[pos2seq_id[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]] * vocab_size) + token_ids[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))])] - (penalties[(pos2seq_id[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] * 3)] + ((convert_float(token_cnt[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))])) * penalties[((pos2seq_id[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] * 3) + 1)]))); float condval; if ((logits[((seq_ids[pos2seq_id[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]] * vocab_size) + token_ids[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))])] < 0.000000e+00f)) { condval = (logits[((seq_ids[pos2seq_id[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]] * vocab_size) + token_ids[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))])] * penalties[((pos2seq_id[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] * 3) + 2)]); } else { condval = (logits[((seq_ids[pos2seq_id[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]] * vocab_size) + token_ids[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))])] / penalties[((pos2seq_id[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] * 3) + 2)]); } logits[((seq_ids[pos2seq_id[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]] * vocab_size) + token_ids[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))])] = condval; } } // Function: gpt_neox_q4f16_1_reshape6_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len) { T_reshape[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] = lv245[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]; } // Function: gpt_neox_q4f16_1_reshape5_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len) { T_reshape[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] = reshape192[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]; } // Function: gpt_neox_q4f16_1_compact_kv_copy_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length) { if (((convert_int(get_group_id(0))) - (batch_size * 8)) < 0) { for (int i = 0; i < (copy_length_indptr[((((convert_int(get_group_id(0))) >> 3) + copy_length_indptr_elem_offset) + 1)] - copy_length_indptr[(((convert_int(get_group_id(0))) >> 3) + copy_length_indptr_elem_offset)]); ++i) { int src_pos = copy_src_dst_pos[((copy_length_indptr[(((convert_int(get_group_id(0))) >> 3) + copy_length_indptr_elem_offset)] + i) + copy_src_dst_pos_elem_offset)]; int dst_pos = copy_src_dst_pos[(((total_copy_length + copy_length_indptr[(((convert_int(get_group_id(0))) >> 3) + copy_length_indptr_elem_offset)]) + i) + copy_src_dst_pos_elem_offset)]; pages[(((((((dst_pos >> 4) * 65536) + (((convert_int(get_group_id(0))) & 7) * 4096)) + (((convert_int(get_local_id(0))) >> 7) * 2048)) + ((dst_pos & 15) * 128)) + pages_elem_offset) + ((convert_int(get_local_id(0))) & 127))] = pages[(((((((src_pos >> 4) * 65536) + (((convert_int(get_group_id(0))) & 7) * 4096)) + (((convert_int(get_local_id(0))) >> 7) * 2048)) + ((src_pos & 15) * 128)) + pages_elem_offset) + ((convert_int(get_local_id(0))) & 127))]; pages[((((((((dst_pos >> 4) * 65536) + (((convert_int(get_group_id(0))) & 7) * 4096)) + (((convert_int(get_local_id(0))) >> 7) * 2048)) + ((dst_pos & 15) * 128)) + pages_elem_offset) + ((convert_int(get_local_id(0))) & 127)) + 32768)] = pages[((((((((src_pos >> 4) * 65536) + (((convert_int(get_group_id(0))) & 7) * 4096)) + (((convert_int(get_local_id(0))) >> 7) * 2048)) + ((src_pos & 15) * 128)) + pages_elem_offset) + ((convert_int(get_local_id(0))) & 127)) + 32768)]; } } } // Function: gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2 #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len) { float8 NT_matmul_intermediate_reindex_pad_local[2]; for (int var = 0; var < 1; ++var) { NT_matmul_intermediate_reindex_pad_local[0] = ((float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)); NT_matmul_intermediate_reindex_pad_local[1] = ((float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)); for (int ax3_0 = 0; ax3_0 < 512; ++ax3_0) { for (int ax3_1 = 0; ax3_1 < 16; ++ax3_1) { uint cse_var_7 = (convert_uint(((ax3_1 & 7) * 4))); half condval; if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < seq_len)) { condval = lv98[(((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + (ax3_0 * 16)) + ax3_1)]; } else { condval = (half)0.000000e+00f; } int8 v_ = (int8)(((((((convert_int(get_group_id(0))) * 262144) + ((convert_int(get_local_id(0))) * 8192)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(1024*0), ((((((convert_int(get_group_id(0))) * 262144) + ((convert_int(get_local_id(0))) * 8192)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(1024*1), ((((((convert_int(get_group_id(0))) * 262144) + ((convert_int(get_local_id(0))) * 8192)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(1024*2), ((((((convert_int(get_group_id(0))) * 262144) + ((convert_int(get_local_id(0))) * 8192)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(1024*3), ((((((convert_int(get_group_id(0))) * 262144) + ((convert_int(get_local_id(0))) * 8192)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(1024*4), ((((((convert_int(get_group_id(0))) * 262144) + ((convert_int(get_local_id(0))) * 8192)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(1024*5), ((((((convert_int(get_group_id(0))) * 262144) + ((convert_int(get_local_id(0))) * 8192)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(1024*6), ((((((convert_int(get_group_id(0))) * 262144) + ((convert_int(get_local_id(0))) * 8192)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(1024*7)); int8 v__1 = (int8)((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 >> 1)))+(256*0), (((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 >> 1)))+(256*1), (((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 >> 1)))+(256*2), (((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 >> 1)))+(256*3), (((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 >> 1)))+(256*4), (((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 >> 1)))+(256*5), (((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 >> 1)))+(256*6), (((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 >> 1)))+(256*7)); NT_matmul_intermediate_reindex_pad_local[0] = (((convert_float8(((half8)(condval, condval, condval, condval, condval, condval, condval, condval)))) * (convert_float8((((convert_half8(((((uint8)(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[v_.s0],gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[v_.s1],gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[v_.s2],gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[v_.s3],gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[v_.s4],gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[v_.s5],gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[v_.s6],gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[v_.s7])) >> ((uint8)(cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7))) & ((uint8)((uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15))))) - ((half8)((half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f))) * ((half8)(gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[v__1.s0],gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[v__1.s1],gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[v__1.s2],gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[v__1.s3],gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[v__1.s4],gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[v__1.s5],gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[v__1.s6],gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[v__1.s7])))))) + NT_matmul_intermediate_reindex_pad_local[0]); half condval_1; if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < seq_len)) { condval_1 = lv98[((((((convert_int(get_group_id(1))) * 65536) + ((convert_int(get_local_id(1))) * 16384)) + (ax3_0 * 16)) + ax3_1) + 8192)]; } else { condval_1 = (half)0.000000e+00f; } int8 v__2 = (int8)(((((((convert_int(get_group_id(0))) * 262144) + ((convert_int(get_local_id(0))) * 8192)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(1024*0), ((((((convert_int(get_group_id(0))) * 262144) + ((convert_int(get_local_id(0))) * 8192)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(1024*1), ((((((convert_int(get_group_id(0))) * 262144) + ((convert_int(get_local_id(0))) * 8192)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(1024*2), ((((((convert_int(get_group_id(0))) * 262144) + ((convert_int(get_local_id(0))) * 8192)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(1024*3), ((((((convert_int(get_group_id(0))) * 262144) + ((convert_int(get_local_id(0))) * 8192)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(1024*4), ((((((convert_int(get_group_id(0))) * 262144) + ((convert_int(get_local_id(0))) * 8192)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(1024*5), ((((((convert_int(get_group_id(0))) * 262144) + ((convert_int(get_local_id(0))) * 8192)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(1024*6), ((((((convert_int(get_group_id(0))) * 262144) + ((convert_int(get_local_id(0))) * 8192)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(1024*7)); int8 v__3 = (int8)((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 >> 1)))+(256*0), (((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 >> 1)))+(256*1), (((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 >> 1)))+(256*2), (((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 >> 1)))+(256*3), (((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 >> 1)))+(256*4), (((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 >> 1)))+(256*5), (((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 >> 1)))+(256*6), (((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 >> 1)))+(256*7)); NT_matmul_intermediate_reindex_pad_local[1] = (((convert_float8(((half8)(condval_1, condval_1, condval_1, condval_1, condval_1, condval_1, condval_1, condval_1)))) * (convert_float8((((convert_half8(((((uint8)(gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[v__2.s0],gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[v__2.s1],gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[v__2.s2],gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[v__2.s3],gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[v__2.s4],gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[v__2.s5],gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[v__2.s6],gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[v__2.s7])) >> ((uint8)(cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7))) & ((uint8)((uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15))))) - ((half8)((half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f))) * ((half8)(gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[v__3.s0],gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[v__3.s1],gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[v__3.s2],gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[v__3.s3],gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[v__3.s4],gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[v__3.s5],gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[v__3.s6],gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[v__3.s7])))))) + NT_matmul_intermediate_reindex_pad_local[1]); } } if ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < seq_len) { vstore8((convert_half8((NT_matmul_intermediate_reindex_pad_local[0] + vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_bias3 + (((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)))))), 0, compute + (((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8))); } if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < seq_len) { vstore8((convert_half8((NT_matmul_intermediate_reindex_pad_local[1] + vload8(0, gpt_neox_layers_0_mlp_dense_4h_to_h_bias3 + (((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)))))), 0, compute + ((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 2048)); } } } // Function: gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2 #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size) { half8 NT_matmul_intermediate_reindex_pad_local[2]; for (int var = 0; var < 1; ++var) { NT_matmul_intermediate_reindex_pad_local[0] = ((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)); NT_matmul_intermediate_reindex_pad_local[1] = ((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)); for (int ax3_0 = 0; ax3_0 < 128; ++ax3_0) { for (int ax3_1 = 0; ax3_1 < 16; ++ax3_1) { uint cse_var_7 = (convert_uint(((ax3_1 & 7) * 4))); half condval; if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { condval = reshape291[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; } else { condval = (half)0.000000e+00f; } int8 v_ = (int8)(((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*0), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*1), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*2), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*3), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*4), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*5), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*6), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*7)); int8 v__1 = (int8)((((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*0), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*1), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*2), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*3), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*4), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*5), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*6), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*7)); NT_matmul_intermediate_reindex_pad_local[0] = (NT_matmul_intermediate_reindex_pad_local[0] + (((half8)(condval, condval, condval, condval, condval, condval, condval, condval)) * (((convert_half8(((((uint8)(gpt_neox_layers_0_attention_dense_q_weight4[v_.s0],gpt_neox_layers_0_attention_dense_q_weight4[v_.s1],gpt_neox_layers_0_attention_dense_q_weight4[v_.s2],gpt_neox_layers_0_attention_dense_q_weight4[v_.s3],gpt_neox_layers_0_attention_dense_q_weight4[v_.s4],gpt_neox_layers_0_attention_dense_q_weight4[v_.s5],gpt_neox_layers_0_attention_dense_q_weight4[v_.s6],gpt_neox_layers_0_attention_dense_q_weight4[v_.s7])) >> ((uint8)(cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7))) & ((uint8)((uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15))))) - ((half8)((half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f))) * ((half8)(gpt_neox_layers_0_attention_dense_q_scale4[v__1.s0],gpt_neox_layers_0_attention_dense_q_scale4[v__1.s1],gpt_neox_layers_0_attention_dense_q_scale4[v__1.s2],gpt_neox_layers_0_attention_dense_q_scale4[v__1.s3],gpt_neox_layers_0_attention_dense_q_scale4[v__1.s4],gpt_neox_layers_0_attention_dense_q_scale4[v__1.s5],gpt_neox_layers_0_attention_dense_q_scale4[v__1.s6],gpt_neox_layers_0_attention_dense_q_scale4[v__1.s7]))))); half condval_1; if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { condval_1 = reshape291[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; } else { condval_1 = (half)0.000000e+00f; } int8 v__2 = (int8)(((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*0), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*1), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*2), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*3), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*4), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*5), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*6), ((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)))+(256*7)); int8 v__3 = (int8)((((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*0), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*1), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*2), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*3), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*4), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*5), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*6), (((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)))+(64*7)); NT_matmul_intermediate_reindex_pad_local[1] = (NT_matmul_intermediate_reindex_pad_local[1] + (((half8)(condval_1, condval_1, condval_1, condval_1, condval_1, condval_1, condval_1, condval_1)) * (((convert_half8(((((uint8)(gpt_neox_layers_0_attention_dense_q_weight4[v__2.s0],gpt_neox_layers_0_attention_dense_q_weight4[v__2.s1],gpt_neox_layers_0_attention_dense_q_weight4[v__2.s2],gpt_neox_layers_0_attention_dense_q_weight4[v__2.s3],gpt_neox_layers_0_attention_dense_q_weight4[v__2.s4],gpt_neox_layers_0_attention_dense_q_weight4[v__2.s5],gpt_neox_layers_0_attention_dense_q_weight4[v__2.s6],gpt_neox_layers_0_attention_dense_q_weight4[v__2.s7])) >> ((uint8)(cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7, cse_var_7))) & ((uint8)((uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15))))) - ((half8)((half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f))) * ((half8)(gpt_neox_layers_0_attention_dense_q_scale4[v__3.s0],gpt_neox_layers_0_attention_dense_q_scale4[v__3.s1],gpt_neox_layers_0_attention_dense_q_scale4[v__3.s2],gpt_neox_layers_0_attention_dense_q_scale4[v__3.s3],gpt_neox_layers_0_attention_dense_q_scale4[v__3.s4],gpt_neox_layers_0_attention_dense_q_scale4[v__3.s5],gpt_neox_layers_0_attention_dense_q_scale4[v__3.s6],gpt_neox_layers_0_attention_dense_q_scale4[v__3.s7]))))); } } if ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size) { vstore8(((vload8(0, lv2 + (((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8))) + (NT_matmul_intermediate_reindex_pad_local[0] + vload8(0, gpt_neox_layers_0_attention_dense_bias4 + (((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8))))) + vload8(0, input_embeds + (((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)))), 0, T_add + (((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8))); } if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size) { vstore8(((vload8(0, lv2 + ((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 2048)) + (NT_matmul_intermediate_reindex_pad_local[1] + vload8(0, gpt_neox_layers_0_attention_dense_bias4 + (((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8))))) + vload8(0, input_embeds + ((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 2048))), 0, T_add + ((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + ((convert_int(get_group_id(0))) * 256)) + ((convert_int(get_local_id(0))) * 8)) + 2048)); } } } // Function: gpt_neox_q4f16_1_apply_bitmask_inplace_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size) { if ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) < (num_seq * vocab_size)) { float condval; if ((((bitmask[((((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % vocab_size) >> 5) + (seq_ids[(((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % (vocab_size * num_seq)) / vocab_size)] * ((vocab_size + 31) >> 5)))] >> (((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % vocab_size) & 31)) & 1) == 1)) { condval = logits[((seq_ids[((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) / vocab_size)] * vocab_size) + ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % vocab_size))]; } else { condval = -3.402823e+38f; } logits[((seq_ids[(((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % (vocab_size * num_seq)) / vocab_size)] * vocab_size) + ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % vocab_size))] = condval; } } // Function: gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1 #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len) { float NT_matmul_intermediate_pad_rf_local[64]; half dequantize_local[16]; float NT_matmul_intermediate_pad_rf_local_1[8]; float NT_matmul_intermediate_pad_local[8]; __local float red_buf0[64]; for (int ax0_1_init = 0; ax0_1_init < 4; ++ax0_1_init) { for (int ax1_fused_2_init = 0; ax1_fused_2_init < 2; ++ax1_fused_2_init) { int8 v_ = (int8)((((ax0_1_init * 2) + ax1_fused_2_init))+(8*0), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*1), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*2), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*3), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*4), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*5), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*6), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*7)); float8 v__1 = (float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f); NT_matmul_intermediate_pad_rf_local[v_.s0] = v__1.s0; NT_matmul_intermediate_pad_rf_local[v_.s1] = v__1.s1; NT_matmul_intermediate_pad_rf_local[v_.s2] = v__1.s2; NT_matmul_intermediate_pad_rf_local[v_.s3] = v__1.s3; NT_matmul_intermediate_pad_rf_local[v_.s4] = v__1.s4; NT_matmul_intermediate_pad_rf_local[v_.s5] = v__1.s5; NT_matmul_intermediate_pad_rf_local[v_.s6] = v__1.s6; NT_matmul_intermediate_pad_rf_local[v_.s7] = v__1.s7; } } for (int ax2_fused_0 = 0; ax2_fused_0 < 32; ++ax2_fused_0) { for (int ax0_0 = 0; ax0_0 < 2; ++ax0_0) { dequantize_local[(ax0_0 * 8)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax0_0 * 1024)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)0) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 1)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax0_0 * 1024)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)4) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 2)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax0_0 * 1024)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)8) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 3)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax0_0 * 1024)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)12) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 4)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax0_0 * 1024)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)16) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 5)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax0_0 * 1024)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)20) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 6)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax0_0 * 1024)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)24) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 7)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3[((((((convert_int(get_group_id(0))) * 4096) + ((convert_int(get_local_id(0))) * 2048)) + (ax0_0 * 1024)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)28) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); } if (((convert_int(get_group_id(1))) * 4) < seq_len) { int8 v__2 = (int8)((0)+(8*0), (0)+(8*1), (0)+(8*2), (0)+(8*3), (0)+(8*4), (0)+(8*5), (0)+(8*6), (0)+(8*7)); float8 v__3 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__2.s0],NT_matmul_intermediate_pad_rf_local[v__2.s1],NT_matmul_intermediate_pad_rf_local[v__2.s2],NT_matmul_intermediate_pad_rf_local[v__2.s3],NT_matmul_intermediate_pad_rf_local[v__2.s4],NT_matmul_intermediate_pad_rf_local[v__2.s5],NT_matmul_intermediate_pad_rf_local[v__2.s6],NT_matmul_intermediate_pad_rf_local[v__2.s7])) + ((convert_float8(vload8(0, lv98 + ((((convert_int(get_group_id(1))) * 32768) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, dequantize_local + 0)))); NT_matmul_intermediate_pad_rf_local[v__2.s0] = v__3.s0; NT_matmul_intermediate_pad_rf_local[v__2.s1] = v__3.s1; NT_matmul_intermediate_pad_rf_local[v__2.s2] = v__3.s2; NT_matmul_intermediate_pad_rf_local[v__2.s3] = v__3.s3; NT_matmul_intermediate_pad_rf_local[v__2.s4] = v__3.s4; NT_matmul_intermediate_pad_rf_local[v__2.s5] = v__3.s5; NT_matmul_intermediate_pad_rf_local[v__2.s6] = v__3.s6; NT_matmul_intermediate_pad_rf_local[v__2.s7] = v__3.s7; int8 v__4 = (int8)((1)+(8*0), (1)+(8*1), (1)+(8*2), (1)+(8*3), (1)+(8*4), (1)+(8*5), (1)+(8*6), (1)+(8*7)); float8 v__5 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__4.s0],NT_matmul_intermediate_pad_rf_local[v__4.s1],NT_matmul_intermediate_pad_rf_local[v__4.s2],NT_matmul_intermediate_pad_rf_local[v__4.s3],NT_matmul_intermediate_pad_rf_local[v__4.s4],NT_matmul_intermediate_pad_rf_local[v__4.s5],NT_matmul_intermediate_pad_rf_local[v__4.s6],NT_matmul_intermediate_pad_rf_local[v__4.s7])) + ((convert_float8(vload8(0, lv98 + ((((convert_int(get_group_id(1))) * 32768) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, dequantize_local + 8)))); NT_matmul_intermediate_pad_rf_local[v__4.s0] = v__5.s0; NT_matmul_intermediate_pad_rf_local[v__4.s1] = v__5.s1; NT_matmul_intermediate_pad_rf_local[v__4.s2] = v__5.s2; NT_matmul_intermediate_pad_rf_local[v__4.s3] = v__5.s3; NT_matmul_intermediate_pad_rf_local[v__4.s4] = v__5.s4; NT_matmul_intermediate_pad_rf_local[v__4.s5] = v__5.s5; NT_matmul_intermediate_pad_rf_local[v__4.s6] = v__5.s6; NT_matmul_intermediate_pad_rf_local[v__4.s7] = v__5.s7; } if ((((convert_int(get_group_id(1))) * 4) + 1) < seq_len) { int8 v__6 = (int8)((2)+(8*0), (2)+(8*1), (2)+(8*2), (2)+(8*3), (2)+(8*4), (2)+(8*5), (2)+(8*6), (2)+(8*7)); float8 v__7 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__6.s0],NT_matmul_intermediate_pad_rf_local[v__6.s1],NT_matmul_intermediate_pad_rf_local[v__6.s2],NT_matmul_intermediate_pad_rf_local[v__6.s3],NT_matmul_intermediate_pad_rf_local[v__6.s4],NT_matmul_intermediate_pad_rf_local[v__6.s5],NT_matmul_intermediate_pad_rf_local[v__6.s6],NT_matmul_intermediate_pad_rf_local[v__6.s7])) + ((convert_float8(vload8(0, lv98 + (((((convert_int(get_group_id(1))) * 32768) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 8192)))) * (convert_float8(vload8(0, dequantize_local + 0)))); NT_matmul_intermediate_pad_rf_local[v__6.s0] = v__7.s0; NT_matmul_intermediate_pad_rf_local[v__6.s1] = v__7.s1; NT_matmul_intermediate_pad_rf_local[v__6.s2] = v__7.s2; NT_matmul_intermediate_pad_rf_local[v__6.s3] = v__7.s3; NT_matmul_intermediate_pad_rf_local[v__6.s4] = v__7.s4; NT_matmul_intermediate_pad_rf_local[v__6.s5] = v__7.s5; NT_matmul_intermediate_pad_rf_local[v__6.s6] = v__7.s6; NT_matmul_intermediate_pad_rf_local[v__6.s7] = v__7.s7; int8 v__8 = (int8)((3)+(8*0), (3)+(8*1), (3)+(8*2), (3)+(8*3), (3)+(8*4), (3)+(8*5), (3)+(8*6), (3)+(8*7)); float8 v__9 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__8.s0],NT_matmul_intermediate_pad_rf_local[v__8.s1],NT_matmul_intermediate_pad_rf_local[v__8.s2],NT_matmul_intermediate_pad_rf_local[v__8.s3],NT_matmul_intermediate_pad_rf_local[v__8.s4],NT_matmul_intermediate_pad_rf_local[v__8.s5],NT_matmul_intermediate_pad_rf_local[v__8.s6],NT_matmul_intermediate_pad_rf_local[v__8.s7])) + ((convert_float8(vload8(0, lv98 + (((((convert_int(get_group_id(1))) * 32768) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 8192)))) * (convert_float8(vload8(0, dequantize_local + 8)))); NT_matmul_intermediate_pad_rf_local[v__8.s0] = v__9.s0; NT_matmul_intermediate_pad_rf_local[v__8.s1] = v__9.s1; NT_matmul_intermediate_pad_rf_local[v__8.s2] = v__9.s2; NT_matmul_intermediate_pad_rf_local[v__8.s3] = v__9.s3; NT_matmul_intermediate_pad_rf_local[v__8.s4] = v__9.s4; NT_matmul_intermediate_pad_rf_local[v__8.s5] = v__9.s5; NT_matmul_intermediate_pad_rf_local[v__8.s6] = v__9.s6; NT_matmul_intermediate_pad_rf_local[v__8.s7] = v__9.s7; } if ((((convert_int(get_group_id(1))) * 4) + 2) < seq_len) { int8 v__10 = (int8)((4)+(8*0), (4)+(8*1), (4)+(8*2), (4)+(8*3), (4)+(8*4), (4)+(8*5), (4)+(8*6), (4)+(8*7)); float8 v__11 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__10.s0],NT_matmul_intermediate_pad_rf_local[v__10.s1],NT_matmul_intermediate_pad_rf_local[v__10.s2],NT_matmul_intermediate_pad_rf_local[v__10.s3],NT_matmul_intermediate_pad_rf_local[v__10.s4],NT_matmul_intermediate_pad_rf_local[v__10.s5],NT_matmul_intermediate_pad_rf_local[v__10.s6],NT_matmul_intermediate_pad_rf_local[v__10.s7])) + ((convert_float8(vload8(0, lv98 + (((((convert_int(get_group_id(1))) * 32768) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 16384)))) * (convert_float8(vload8(0, dequantize_local + 0)))); NT_matmul_intermediate_pad_rf_local[v__10.s0] = v__11.s0; NT_matmul_intermediate_pad_rf_local[v__10.s1] = v__11.s1; NT_matmul_intermediate_pad_rf_local[v__10.s2] = v__11.s2; NT_matmul_intermediate_pad_rf_local[v__10.s3] = v__11.s3; NT_matmul_intermediate_pad_rf_local[v__10.s4] = v__11.s4; NT_matmul_intermediate_pad_rf_local[v__10.s5] = v__11.s5; NT_matmul_intermediate_pad_rf_local[v__10.s6] = v__11.s6; NT_matmul_intermediate_pad_rf_local[v__10.s7] = v__11.s7; int8 v__12 = (int8)((5)+(8*0), (5)+(8*1), (5)+(8*2), (5)+(8*3), (5)+(8*4), (5)+(8*5), (5)+(8*6), (5)+(8*7)); float8 v__13 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__12.s0],NT_matmul_intermediate_pad_rf_local[v__12.s1],NT_matmul_intermediate_pad_rf_local[v__12.s2],NT_matmul_intermediate_pad_rf_local[v__12.s3],NT_matmul_intermediate_pad_rf_local[v__12.s4],NT_matmul_intermediate_pad_rf_local[v__12.s5],NT_matmul_intermediate_pad_rf_local[v__12.s6],NT_matmul_intermediate_pad_rf_local[v__12.s7])) + ((convert_float8(vload8(0, lv98 + (((((convert_int(get_group_id(1))) * 32768) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 16384)))) * (convert_float8(vload8(0, dequantize_local + 8)))); NT_matmul_intermediate_pad_rf_local[v__12.s0] = v__13.s0; NT_matmul_intermediate_pad_rf_local[v__12.s1] = v__13.s1; NT_matmul_intermediate_pad_rf_local[v__12.s2] = v__13.s2; NT_matmul_intermediate_pad_rf_local[v__12.s3] = v__13.s3; NT_matmul_intermediate_pad_rf_local[v__12.s4] = v__13.s4; NT_matmul_intermediate_pad_rf_local[v__12.s5] = v__13.s5; NT_matmul_intermediate_pad_rf_local[v__12.s6] = v__13.s6; NT_matmul_intermediate_pad_rf_local[v__12.s7] = v__13.s7; } if ((((convert_int(get_group_id(1))) * 4) + 3) < seq_len) { int8 v__14 = (int8)((6)+(8*0), (6)+(8*1), (6)+(8*2), (6)+(8*3), (6)+(8*4), (6)+(8*5), (6)+(8*6), (6)+(8*7)); float8 v__15 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__14.s0],NT_matmul_intermediate_pad_rf_local[v__14.s1],NT_matmul_intermediate_pad_rf_local[v__14.s2],NT_matmul_intermediate_pad_rf_local[v__14.s3],NT_matmul_intermediate_pad_rf_local[v__14.s4],NT_matmul_intermediate_pad_rf_local[v__14.s5],NT_matmul_intermediate_pad_rf_local[v__14.s6],NT_matmul_intermediate_pad_rf_local[v__14.s7])) + ((convert_float8(vload8(0, lv98 + (((((convert_int(get_group_id(1))) * 32768) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 24576)))) * (convert_float8(vload8(0, dequantize_local + 0)))); NT_matmul_intermediate_pad_rf_local[v__14.s0] = v__15.s0; NT_matmul_intermediate_pad_rf_local[v__14.s1] = v__15.s1; NT_matmul_intermediate_pad_rf_local[v__14.s2] = v__15.s2; NT_matmul_intermediate_pad_rf_local[v__14.s3] = v__15.s3; NT_matmul_intermediate_pad_rf_local[v__14.s4] = v__15.s4; NT_matmul_intermediate_pad_rf_local[v__14.s5] = v__15.s5; NT_matmul_intermediate_pad_rf_local[v__14.s6] = v__15.s6; NT_matmul_intermediate_pad_rf_local[v__14.s7] = v__15.s7; int8 v__16 = (int8)((7)+(8*0), (7)+(8*1), (7)+(8*2), (7)+(8*3), (7)+(8*4), (7)+(8*5), (7)+(8*6), (7)+(8*7)); float8 v__17 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__16.s0],NT_matmul_intermediate_pad_rf_local[v__16.s1],NT_matmul_intermediate_pad_rf_local[v__16.s2],NT_matmul_intermediate_pad_rf_local[v__16.s3],NT_matmul_intermediate_pad_rf_local[v__16.s4],NT_matmul_intermediate_pad_rf_local[v__16.s5],NT_matmul_intermediate_pad_rf_local[v__16.s6],NT_matmul_intermediate_pad_rf_local[v__16.s7])) + ((convert_float8(vload8(0, lv98 + (((((convert_int(get_group_id(1))) * 32768) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 24576)))) * (convert_float8(vload8(0, dequantize_local + 8)))); NT_matmul_intermediate_pad_rf_local[v__16.s0] = v__17.s0; NT_matmul_intermediate_pad_rf_local[v__16.s1] = v__17.s1; NT_matmul_intermediate_pad_rf_local[v__16.s2] = v__17.s2; NT_matmul_intermediate_pad_rf_local[v__16.s3] = v__17.s3; NT_matmul_intermediate_pad_rf_local[v__16.s4] = v__17.s4; NT_matmul_intermediate_pad_rf_local[v__16.s5] = v__17.s5; NT_matmul_intermediate_pad_rf_local[v__16.s6] = v__17.s6; NT_matmul_intermediate_pad_rf_local[v__16.s7] = v__17.s7; } } for (int ax3_fused_2_0 = 0; ax3_fused_2_0 < 1; ++ax3_fused_2_0) { for (int ax2 = 0; ax2 < 4; ++ax2) { vstore2(((float2)(0.000000e+00f, 0.000000e+00f)), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + (ax2 * 2))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 8))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 16))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 24))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 32))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 40))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 48))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 56))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); } } for (int ax2_fused_2 = 0; ax2_fused_2 < 2; ++ax2_fused_2) { for (int ax1 = 0; ax1 < 4; ++ax1) { barrier(CLK_LOCAL_MEM_FENCE); ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_intermediate_pad_rf_local_1[((ax1 * 2) + ax2_fused_2)]; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 16) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 8) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 4) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 2) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); } barrier(CLK_LOCAL_MEM_FENCE); NT_matmul_intermediate_pad_local[((ax1 * 2) + ax2_fused_2)] = ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) * 32)]; } } for (int ax0 = 0; ax0 < 4; ++ax0) { for (int ax1_fused_2 = 0; ax1_fused_2 < 2; ++ax1_fused_2) { if (((bool)(((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)(((bool)(((convert_int(get_group_id(1))) - ((seq_len + 3) >> 2)) < 0)) || ((bool)((convert_int(get_group_id(1))) == 0)))))) && ((bool)((((convert_int(get_group_id(1))) * 4) + ax0) < seq_len))) { compute[((((((convert_int(get_group_id(1))) * 8192) + (ax0 * 2048)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] = (convert_half((NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_mlp_dense_4h_to_h_bias3[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]))); } } } } // Function: gpt_neox_q4f16_1_layer_norm2_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed) { float in_thread_input_embed_red_temp_v0_shared[1]; float in_thread_input_embed_red_temp_v1_shared[1]; __local float red_buf0[256]; __local float red_buf1[256]; __local float input_embed_red_temp_v0_shared[1]; __local float input_embed_red_temp_v1_shared[1]; in_thread_input_embed_red_temp_v0_shared[0] = 0.000000e+00f; in_thread_input_embed_red_temp_v1_shared[0] = 0.000000e+00f; float v_input_embed_red_temp_v0 = (in_thread_input_embed_red_temp_v0_shared[0] + (convert_float(input_embed[(convert_int(get_local_id(0)))]))); float v_input_embed_red_temp_v1 = (in_thread_input_embed_red_temp_v1_shared[0] + ((convert_float(input_embed[(convert_int(get_local_id(0)))])) * (convert_float(input_embed[(convert_int(get_local_id(0)))])))); in_thread_input_embed_red_temp_v0_shared[0] = v_input_embed_red_temp_v0; in_thread_input_embed_red_temp_v1_shared[0] = v_input_embed_red_temp_v1; float v_input_embed_red_temp_v0_1 = (in_thread_input_embed_red_temp_v0_shared[0] + (convert_float(input_embed[((convert_int(get_local_id(0))) + 256)]))); float v_input_embed_red_temp_v1_1 = (in_thread_input_embed_red_temp_v1_shared[0] + ((convert_float(input_embed[((convert_int(get_local_id(0))) + 256)])) * (convert_float(input_embed[((convert_int(get_local_id(0))) + 256)])))); in_thread_input_embed_red_temp_v0_shared[0] = v_input_embed_red_temp_v0_1; in_thread_input_embed_red_temp_v1_shared[0] = v_input_embed_red_temp_v1_1; float v_input_embed_red_temp_v0_2 = (in_thread_input_embed_red_temp_v0_shared[0] + (convert_float(input_embed[((convert_int(get_local_id(0))) + 512)]))); float v_input_embed_red_temp_v1_2 = (in_thread_input_embed_red_temp_v1_shared[0] + ((convert_float(input_embed[((convert_int(get_local_id(0))) + 512)])) * (convert_float(input_embed[((convert_int(get_local_id(0))) + 512)])))); in_thread_input_embed_red_temp_v0_shared[0] = v_input_embed_red_temp_v0_2; in_thread_input_embed_red_temp_v1_shared[0] = v_input_embed_red_temp_v1_2; float v_input_embed_red_temp_v0_3 = (in_thread_input_embed_red_temp_v0_shared[0] + (convert_float(input_embed[((convert_int(get_local_id(0))) + 768)]))); float v_input_embed_red_temp_v1_3 = (in_thread_input_embed_red_temp_v1_shared[0] + ((convert_float(input_embed[((convert_int(get_local_id(0))) + 768)])) * (convert_float(input_embed[((convert_int(get_local_id(0))) + 768)])))); in_thread_input_embed_red_temp_v0_shared[0] = v_input_embed_red_temp_v0_3; in_thread_input_embed_red_temp_v1_shared[0] = v_input_embed_red_temp_v1_3; float v_input_embed_red_temp_v0_4 = (in_thread_input_embed_red_temp_v0_shared[0] + (convert_float(input_embed[((convert_int(get_local_id(0))) + 1024)]))); float v_input_embed_red_temp_v1_4 = (in_thread_input_embed_red_temp_v1_shared[0] + ((convert_float(input_embed[((convert_int(get_local_id(0))) + 1024)])) * (convert_float(input_embed[((convert_int(get_local_id(0))) + 1024)])))); in_thread_input_embed_red_temp_v0_shared[0] = v_input_embed_red_temp_v0_4; in_thread_input_embed_red_temp_v1_shared[0] = v_input_embed_red_temp_v1_4; float v_input_embed_red_temp_v0_5 = (in_thread_input_embed_red_temp_v0_shared[0] + (convert_float(input_embed[((convert_int(get_local_id(0))) + 1280)]))); float v_input_embed_red_temp_v1_5 = (in_thread_input_embed_red_temp_v1_shared[0] + ((convert_float(input_embed[((convert_int(get_local_id(0))) + 1280)])) * (convert_float(input_embed[((convert_int(get_local_id(0))) + 1280)])))); in_thread_input_embed_red_temp_v0_shared[0] = v_input_embed_red_temp_v0_5; in_thread_input_embed_red_temp_v1_shared[0] = v_input_embed_red_temp_v1_5; float v_input_embed_red_temp_v0_6 = (in_thread_input_embed_red_temp_v0_shared[0] + (convert_float(input_embed[((convert_int(get_local_id(0))) + 1536)]))); float v_input_embed_red_temp_v1_6 = (in_thread_input_embed_red_temp_v1_shared[0] + ((convert_float(input_embed[((convert_int(get_local_id(0))) + 1536)])) * (convert_float(input_embed[((convert_int(get_local_id(0))) + 1536)])))); in_thread_input_embed_red_temp_v0_shared[0] = v_input_embed_red_temp_v0_6; in_thread_input_embed_red_temp_v1_shared[0] = v_input_embed_red_temp_v1_6; float v_input_embed_red_temp_v0_7 = (in_thread_input_embed_red_temp_v0_shared[0] + (convert_float(input_embed[((convert_int(get_local_id(0))) + 1792)]))); float v_input_embed_red_temp_v1_7 = (in_thread_input_embed_red_temp_v1_shared[0] + ((convert_float(input_embed[((convert_int(get_local_id(0))) + 1792)])) * (convert_float(input_embed[((convert_int(get_local_id(0))) + 1792)])))); in_thread_input_embed_red_temp_v0_shared[0] = v_input_embed_red_temp_v0_7; in_thread_input_embed_red_temp_v1_shared[0] = v_input_embed_red_temp_v1_7; barrier(CLK_LOCAL_MEM_FENCE); ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = in_thread_input_embed_red_temp_v0_shared[0]; ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = in_thread_input_embed_red_temp_v1_shared[0]; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 128) { ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 128)]); ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 128)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 64) { ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 64)]); ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 64)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 32) { ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 32)]); ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 32)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 16) { ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 16)]); ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 16)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 8) { ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 8)]); ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 8)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 4) { ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 4)]); ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 4)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 2) { ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 2)]); ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 2)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 1) { ((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf0)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) + 1)]); ((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] = (((volatile __local float*)red_buf1)[(convert_int(get_local_id(0)))] + ((volatile __local float*)red_buf1)[((convert_int(get_local_id(0))) + 1)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) == 0) { input_embed_red_temp_v0_shared[0] = ((volatile __local float*)red_buf0)[0]; input_embed_red_temp_v1_shared[0] = ((volatile __local float*)red_buf1)[0]; } barrier(CLK_LOCAL_MEM_FENCE); T_layer_norm[(convert_int(get_local_id(0)))] = (((convert_half((((convert_float(input_embed[(convert_int(get_local_id(0)))])) - (input_embed_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embed_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embed_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embed_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight2[(convert_int(get_local_id(0)))]) + gpt_neox_layers_0_input_layernorm_bias2[(convert_int(get_local_id(0)))]); T_layer_norm[((convert_int(get_local_id(0))) + 256)] = (((convert_half((((convert_float(input_embed[((convert_int(get_local_id(0))) + 256)])) - (input_embed_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embed_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embed_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embed_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight2[((convert_int(get_local_id(0))) + 256)]) + gpt_neox_layers_0_input_layernorm_bias2[((convert_int(get_local_id(0))) + 256)]); T_layer_norm[((convert_int(get_local_id(0))) + 512)] = (((convert_half((((convert_float(input_embed[((convert_int(get_local_id(0))) + 512)])) - (input_embed_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embed_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embed_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embed_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight2[((convert_int(get_local_id(0))) + 512)]) + gpt_neox_layers_0_input_layernorm_bias2[((convert_int(get_local_id(0))) + 512)]); T_layer_norm[((convert_int(get_local_id(0))) + 768)] = (((convert_half((((convert_float(input_embed[((convert_int(get_local_id(0))) + 768)])) - (input_embed_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embed_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embed_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embed_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight2[((convert_int(get_local_id(0))) + 768)]) + gpt_neox_layers_0_input_layernorm_bias2[((convert_int(get_local_id(0))) + 768)]); T_layer_norm[((convert_int(get_local_id(0))) + 1024)] = (((convert_half((((convert_float(input_embed[((convert_int(get_local_id(0))) + 1024)])) - (input_embed_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embed_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embed_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embed_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight2[((convert_int(get_local_id(0))) + 1024)]) + gpt_neox_layers_0_input_layernorm_bias2[((convert_int(get_local_id(0))) + 1024)]); T_layer_norm[((convert_int(get_local_id(0))) + 1280)] = (((convert_half((((convert_float(input_embed[((convert_int(get_local_id(0))) + 1280)])) - (input_embed_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embed_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embed_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embed_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight2[((convert_int(get_local_id(0))) + 1280)]) + gpt_neox_layers_0_input_layernorm_bias2[((convert_int(get_local_id(0))) + 1280)]); T_layer_norm[((convert_int(get_local_id(0))) + 1536)] = (((convert_half((((convert_float(input_embed[((convert_int(get_local_id(0))) + 1536)])) - (input_embed_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embed_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embed_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embed_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight2[((convert_int(get_local_id(0))) + 1536)]) + gpt_neox_layers_0_input_layernorm_bias2[((convert_int(get_local_id(0))) + 1536)]); T_layer_norm[((convert_int(get_local_id(0))) + 1792)] = (((convert_half((((convert_float(input_embed[((convert_int(get_local_id(0))) + 1792)])) - (input_embed_red_temp_v0_shared[0] * 4.882812e-04f)) * (1.000000e+00f / sqrt((((input_embed_red_temp_v1_shared[0] * 4.882812e-04f) - ((input_embed_red_temp_v0_shared[0] * 4.882812e-04f) * (input_embed_red_temp_v0_shared[0] * 4.882812e-04f))) + 1.000000e-05f)))))) * gpt_neox_layers_0_input_layernorm_weight2[((convert_int(get_local_id(0))) + 1792)]) + gpt_neox_layers_0_input_layernorm_bias2[((convert_int(get_local_id(0))) + 1792)]); } // Function: gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size) { half NT_matmul_intermediate_pad_rf_local[32]; half dequantize_local[16]; half NT_matmul_intermediate_pad_rf_local_1[4]; half NT_matmul_intermediate_pad_local[4]; __local half red_buf0[64]; for (int ax0_1_init = 0; ax0_1_init < 2; ++ax0_1_init) { for (int ax1_fused_2_init = 0; ax1_fused_2_init < 2; ++ax1_fused_2_init) { int8 v_ = (int8)((((ax0_1_init * 2) + ax1_fused_2_init))+(4*0), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*1), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*2), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*3), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*4), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*5), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*6), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*7)); half8 v__1 = (half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f); NT_matmul_intermediate_pad_rf_local[v_.s0] = v__1.s0; NT_matmul_intermediate_pad_rf_local[v_.s1] = v__1.s1; NT_matmul_intermediate_pad_rf_local[v_.s2] = v__1.s2; NT_matmul_intermediate_pad_rf_local[v_.s3] = v__1.s3; NT_matmul_intermediate_pad_rf_local[v_.s4] = v__1.s4; NT_matmul_intermediate_pad_rf_local[v_.s5] = v__1.s5; NT_matmul_intermediate_pad_rf_local[v_.s6] = v__1.s6; NT_matmul_intermediate_pad_rf_local[v_.s7] = v__1.s7; } } for (int ax2_fused_0 = 0; ax2_fused_0 < 8; ++ax2_fused_0) { for (int ax0_0 = 0; ax0_0 < 2; ++ax0_0) { dequantize_local[(ax0_0 * 8)] = (((convert_half(((gpt_neox_layers_0_attention_query_key_value_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)0) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_query_key_value_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 1)] = (((convert_half(((gpt_neox_layers_0_attention_query_key_value_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)4) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_query_key_value_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 2)] = (((convert_half(((gpt_neox_layers_0_attention_query_key_value_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)8) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_query_key_value_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 3)] = (((convert_half(((gpt_neox_layers_0_attention_query_key_value_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)12) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_query_key_value_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 4)] = (((convert_half(((gpt_neox_layers_0_attention_query_key_value_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)16) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_query_key_value_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 5)] = (((convert_half(((gpt_neox_layers_0_attention_query_key_value_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)20) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_query_key_value_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 6)] = (((convert_half(((gpt_neox_layers_0_attention_query_key_value_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)24) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_query_key_value_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 7)] = (((convert_half(((gpt_neox_layers_0_attention_query_key_value_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)28) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_query_key_value_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); } if (((convert_int(get_group_id(1))) * 2) < batch_size) { int8 v__2 = (int8)((0)+(4*0), (0)+(4*1), (0)+(4*2), (0)+(4*3), (0)+(4*4), (0)+(4*5), (0)+(4*6), (0)+(4*7)); half8 v__3 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__2.s0],NT_matmul_intermediate_pad_rf_local[v__2.s1],NT_matmul_intermediate_pad_rf_local[v__2.s2],NT_matmul_intermediate_pad_rf_local[v__2.s3],NT_matmul_intermediate_pad_rf_local[v__2.s4],NT_matmul_intermediate_pad_rf_local[v__2.s5],NT_matmul_intermediate_pad_rf_local[v__2.s6],NT_matmul_intermediate_pad_rf_local[v__2.s7])) + (vload8(0, layer_norm147 + ((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, dequantize_local + 0)); NT_matmul_intermediate_pad_rf_local[v__2.s0] = v__3.s0; NT_matmul_intermediate_pad_rf_local[v__2.s1] = v__3.s1; NT_matmul_intermediate_pad_rf_local[v__2.s2] = v__3.s2; NT_matmul_intermediate_pad_rf_local[v__2.s3] = v__3.s3; NT_matmul_intermediate_pad_rf_local[v__2.s4] = v__3.s4; NT_matmul_intermediate_pad_rf_local[v__2.s5] = v__3.s5; NT_matmul_intermediate_pad_rf_local[v__2.s6] = v__3.s6; NT_matmul_intermediate_pad_rf_local[v__2.s7] = v__3.s7; int8 v__4 = (int8)((1)+(4*0), (1)+(4*1), (1)+(4*2), (1)+(4*3), (1)+(4*4), (1)+(4*5), (1)+(4*6), (1)+(4*7)); half8 v__5 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__4.s0],NT_matmul_intermediate_pad_rf_local[v__4.s1],NT_matmul_intermediate_pad_rf_local[v__4.s2],NT_matmul_intermediate_pad_rf_local[v__4.s3],NT_matmul_intermediate_pad_rf_local[v__4.s4],NT_matmul_intermediate_pad_rf_local[v__4.s5],NT_matmul_intermediate_pad_rf_local[v__4.s6],NT_matmul_intermediate_pad_rf_local[v__4.s7])) + (vload8(0, layer_norm147 + ((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, dequantize_local + 8)); NT_matmul_intermediate_pad_rf_local[v__4.s0] = v__5.s0; NT_matmul_intermediate_pad_rf_local[v__4.s1] = v__5.s1; NT_matmul_intermediate_pad_rf_local[v__4.s2] = v__5.s2; NT_matmul_intermediate_pad_rf_local[v__4.s3] = v__5.s3; NT_matmul_intermediate_pad_rf_local[v__4.s4] = v__5.s4; NT_matmul_intermediate_pad_rf_local[v__4.s5] = v__5.s5; NT_matmul_intermediate_pad_rf_local[v__4.s6] = v__5.s6; NT_matmul_intermediate_pad_rf_local[v__4.s7] = v__5.s7; } if ((((convert_int(get_group_id(1))) * 2) + 1) < batch_size) { int8 v__6 = (int8)((2)+(4*0), (2)+(4*1), (2)+(4*2), (2)+(4*3), (2)+(4*4), (2)+(4*5), (2)+(4*6), (2)+(4*7)); half8 v__7 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__6.s0],NT_matmul_intermediate_pad_rf_local[v__6.s1],NT_matmul_intermediate_pad_rf_local[v__6.s2],NT_matmul_intermediate_pad_rf_local[v__6.s3],NT_matmul_intermediate_pad_rf_local[v__6.s4],NT_matmul_intermediate_pad_rf_local[v__6.s5],NT_matmul_intermediate_pad_rf_local[v__6.s6],NT_matmul_intermediate_pad_rf_local[v__6.s7])) + (vload8(0, layer_norm147 + (((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)) * vload8(0, dequantize_local + 0)); NT_matmul_intermediate_pad_rf_local[v__6.s0] = v__7.s0; NT_matmul_intermediate_pad_rf_local[v__6.s1] = v__7.s1; NT_matmul_intermediate_pad_rf_local[v__6.s2] = v__7.s2; NT_matmul_intermediate_pad_rf_local[v__6.s3] = v__7.s3; NT_matmul_intermediate_pad_rf_local[v__6.s4] = v__7.s4; NT_matmul_intermediate_pad_rf_local[v__6.s5] = v__7.s5; NT_matmul_intermediate_pad_rf_local[v__6.s6] = v__7.s6; NT_matmul_intermediate_pad_rf_local[v__6.s7] = v__7.s7; int8 v__8 = (int8)((3)+(4*0), (3)+(4*1), (3)+(4*2), (3)+(4*3), (3)+(4*4), (3)+(4*5), (3)+(4*6), (3)+(4*7)); half8 v__9 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__8.s0],NT_matmul_intermediate_pad_rf_local[v__8.s1],NT_matmul_intermediate_pad_rf_local[v__8.s2],NT_matmul_intermediate_pad_rf_local[v__8.s3],NT_matmul_intermediate_pad_rf_local[v__8.s4],NT_matmul_intermediate_pad_rf_local[v__8.s5],NT_matmul_intermediate_pad_rf_local[v__8.s6],NT_matmul_intermediate_pad_rf_local[v__8.s7])) + (vload8(0, layer_norm147 + (((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)) * vload8(0, dequantize_local + 8)); NT_matmul_intermediate_pad_rf_local[v__8.s0] = v__9.s0; NT_matmul_intermediate_pad_rf_local[v__8.s1] = v__9.s1; NT_matmul_intermediate_pad_rf_local[v__8.s2] = v__9.s2; NT_matmul_intermediate_pad_rf_local[v__8.s3] = v__9.s3; NT_matmul_intermediate_pad_rf_local[v__8.s4] = v__9.s4; NT_matmul_intermediate_pad_rf_local[v__8.s5] = v__9.s5; NT_matmul_intermediate_pad_rf_local[v__8.s6] = v__9.s6; NT_matmul_intermediate_pad_rf_local[v__8.s7] = v__9.s7; } } for (int ax3_fused_2_0 = 0; ax3_fused_2_0 < 1; ++ax3_fused_2_0) { for (int ax2 = 0; ax2 < 2; ++ax2) { vstore2(((half2)((half)0.000000e+00f, (half)0.000000e+00f)), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + (ax2 * 2))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 4))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 8))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 12))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 16))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 20))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 24))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 28))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); } } for (int ax2_fused_2 = 0; ax2_fused_2 < 2; ++ax2_fused_2) { for (int ax1 = 0; ax1 < 2; ++ax1) { barrier(CLK_LOCAL_MEM_FENCE); ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_intermediate_pad_rf_local_1[((ax1 * 2) + ax2_fused_2)]; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 16) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 8) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 4) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 2) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); } barrier(CLK_LOCAL_MEM_FENCE); NT_matmul_intermediate_pad_local[((ax1 * 2) + ax2_fused_2)] = ((volatile __local half*)red_buf0)[((convert_int(get_local_id(0))) * 32)]; } } for (int ax0 = 0; ax0 < 2; ++ax0) { for (int ax1_fused_2 = 0; ax1_fused_2 < 2; ++ax1_fused_2) { if (((bool)(((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)(((bool)(((convert_int(get_group_id(1))) - ((batch_size + 1) >> 1)) < 0)) || ((bool)((convert_int(get_group_id(1))) == 0)))))) && ((bool)((((convert_int(get_group_id(1))) * 2) + ax0) < batch_size))) { T_add[((((((convert_int(get_group_id(1))) * 12288) + (ax0 * 6144)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] = (NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_attention_query_key_value_bias4[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]); } } } } // Function: gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len) { half NT_matmul_intermediate_pad_rf_local[32]; half dequantize_local[16]; half NT_matmul_intermediate_pad_rf_local_1[4]; half NT_matmul_intermediate_pad_local[4]; __local half red_buf0[64]; for (int ax0_1_init = 0; ax0_1_init < 2; ++ax0_1_init) { for (int ax1_fused_2_init = 0; ax1_fused_2_init < 2; ++ax1_fused_2_init) { int8 v_ = (int8)((((ax0_1_init * 2) + ax1_fused_2_init))+(4*0), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*1), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*2), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*3), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*4), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*5), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*6), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*7)); half8 v__1 = (half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f); NT_matmul_intermediate_pad_rf_local[v_.s0] = v__1.s0; NT_matmul_intermediate_pad_rf_local[v_.s1] = v__1.s1; NT_matmul_intermediate_pad_rf_local[v_.s2] = v__1.s2; NT_matmul_intermediate_pad_rf_local[v_.s3] = v__1.s3; NT_matmul_intermediate_pad_rf_local[v_.s4] = v__1.s4; NT_matmul_intermediate_pad_rf_local[v_.s5] = v__1.s5; NT_matmul_intermediate_pad_rf_local[v_.s6] = v__1.s6; NT_matmul_intermediate_pad_rf_local[v_.s7] = v__1.s7; } } for (int ax2_fused_0 = 0; ax2_fused_0 < 8; ++ax2_fused_0) { for (int ax0_0 = 0; ax0_0 < 2; ++ax0_0) { dequantize_local[(ax0_0 * 8)] = (((convert_half(((gpt_neox_layers_0_attention_query_key_value_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)0) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_query_key_value_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 1)] = (((convert_half(((gpt_neox_layers_0_attention_query_key_value_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)4) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_query_key_value_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 2)] = (((convert_half(((gpt_neox_layers_0_attention_query_key_value_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)8) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_query_key_value_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 3)] = (((convert_half(((gpt_neox_layers_0_attention_query_key_value_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)12) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_query_key_value_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 4)] = (((convert_half(((gpt_neox_layers_0_attention_query_key_value_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)16) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_query_key_value_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 5)] = (((convert_half(((gpt_neox_layers_0_attention_query_key_value_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)20) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_query_key_value_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 6)] = (((convert_half(((gpt_neox_layers_0_attention_query_key_value_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)24) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_query_key_value_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 7)] = (((convert_half(((gpt_neox_layers_0_attention_query_key_value_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)28) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_query_key_value_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); } if (((convert_int(get_group_id(1))) * 2) < seq_len) { int8 v__2 = (int8)((0)+(4*0), (0)+(4*1), (0)+(4*2), (0)+(4*3), (0)+(4*4), (0)+(4*5), (0)+(4*6), (0)+(4*7)); half8 v__3 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__2.s0],NT_matmul_intermediate_pad_rf_local[v__2.s1],NT_matmul_intermediate_pad_rf_local[v__2.s2],NT_matmul_intermediate_pad_rf_local[v__2.s3],NT_matmul_intermediate_pad_rf_local[v__2.s4],NT_matmul_intermediate_pad_rf_local[v__2.s5],NT_matmul_intermediate_pad_rf_local[v__2.s6],NT_matmul_intermediate_pad_rf_local[v__2.s7])) + (vload8(0, layer_norm98 + ((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, dequantize_local + 0)); NT_matmul_intermediate_pad_rf_local[v__2.s0] = v__3.s0; NT_matmul_intermediate_pad_rf_local[v__2.s1] = v__3.s1; NT_matmul_intermediate_pad_rf_local[v__2.s2] = v__3.s2; NT_matmul_intermediate_pad_rf_local[v__2.s3] = v__3.s3; NT_matmul_intermediate_pad_rf_local[v__2.s4] = v__3.s4; NT_matmul_intermediate_pad_rf_local[v__2.s5] = v__3.s5; NT_matmul_intermediate_pad_rf_local[v__2.s6] = v__3.s6; NT_matmul_intermediate_pad_rf_local[v__2.s7] = v__3.s7; int8 v__4 = (int8)((1)+(4*0), (1)+(4*1), (1)+(4*2), (1)+(4*3), (1)+(4*4), (1)+(4*5), (1)+(4*6), (1)+(4*7)); half8 v__5 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__4.s0],NT_matmul_intermediate_pad_rf_local[v__4.s1],NT_matmul_intermediate_pad_rf_local[v__4.s2],NT_matmul_intermediate_pad_rf_local[v__4.s3],NT_matmul_intermediate_pad_rf_local[v__4.s4],NT_matmul_intermediate_pad_rf_local[v__4.s5],NT_matmul_intermediate_pad_rf_local[v__4.s6],NT_matmul_intermediate_pad_rf_local[v__4.s7])) + (vload8(0, layer_norm98 + ((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, dequantize_local + 8)); NT_matmul_intermediate_pad_rf_local[v__4.s0] = v__5.s0; NT_matmul_intermediate_pad_rf_local[v__4.s1] = v__5.s1; NT_matmul_intermediate_pad_rf_local[v__4.s2] = v__5.s2; NT_matmul_intermediate_pad_rf_local[v__4.s3] = v__5.s3; NT_matmul_intermediate_pad_rf_local[v__4.s4] = v__5.s4; NT_matmul_intermediate_pad_rf_local[v__4.s5] = v__5.s5; NT_matmul_intermediate_pad_rf_local[v__4.s6] = v__5.s6; NT_matmul_intermediate_pad_rf_local[v__4.s7] = v__5.s7; } if ((((convert_int(get_group_id(1))) * 2) + 1) < seq_len) { int8 v__6 = (int8)((2)+(4*0), (2)+(4*1), (2)+(4*2), (2)+(4*3), (2)+(4*4), (2)+(4*5), (2)+(4*6), (2)+(4*7)); half8 v__7 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__6.s0],NT_matmul_intermediate_pad_rf_local[v__6.s1],NT_matmul_intermediate_pad_rf_local[v__6.s2],NT_matmul_intermediate_pad_rf_local[v__6.s3],NT_matmul_intermediate_pad_rf_local[v__6.s4],NT_matmul_intermediate_pad_rf_local[v__6.s5],NT_matmul_intermediate_pad_rf_local[v__6.s6],NT_matmul_intermediate_pad_rf_local[v__6.s7])) + (vload8(0, layer_norm98 + (((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)) * vload8(0, dequantize_local + 0)); NT_matmul_intermediate_pad_rf_local[v__6.s0] = v__7.s0; NT_matmul_intermediate_pad_rf_local[v__6.s1] = v__7.s1; NT_matmul_intermediate_pad_rf_local[v__6.s2] = v__7.s2; NT_matmul_intermediate_pad_rf_local[v__6.s3] = v__7.s3; NT_matmul_intermediate_pad_rf_local[v__6.s4] = v__7.s4; NT_matmul_intermediate_pad_rf_local[v__6.s5] = v__7.s5; NT_matmul_intermediate_pad_rf_local[v__6.s6] = v__7.s6; NT_matmul_intermediate_pad_rf_local[v__6.s7] = v__7.s7; int8 v__8 = (int8)((3)+(4*0), (3)+(4*1), (3)+(4*2), (3)+(4*3), (3)+(4*4), (3)+(4*5), (3)+(4*6), (3)+(4*7)); half8 v__9 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__8.s0],NT_matmul_intermediate_pad_rf_local[v__8.s1],NT_matmul_intermediate_pad_rf_local[v__8.s2],NT_matmul_intermediate_pad_rf_local[v__8.s3],NT_matmul_intermediate_pad_rf_local[v__8.s4],NT_matmul_intermediate_pad_rf_local[v__8.s5],NT_matmul_intermediate_pad_rf_local[v__8.s6],NT_matmul_intermediate_pad_rf_local[v__8.s7])) + (vload8(0, layer_norm98 + (((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)) * vload8(0, dequantize_local + 8)); NT_matmul_intermediate_pad_rf_local[v__8.s0] = v__9.s0; NT_matmul_intermediate_pad_rf_local[v__8.s1] = v__9.s1; NT_matmul_intermediate_pad_rf_local[v__8.s2] = v__9.s2; NT_matmul_intermediate_pad_rf_local[v__8.s3] = v__9.s3; NT_matmul_intermediate_pad_rf_local[v__8.s4] = v__9.s4; NT_matmul_intermediate_pad_rf_local[v__8.s5] = v__9.s5; NT_matmul_intermediate_pad_rf_local[v__8.s6] = v__9.s6; NT_matmul_intermediate_pad_rf_local[v__8.s7] = v__9.s7; } } for (int ax3_fused_2_0 = 0; ax3_fused_2_0 < 1; ++ax3_fused_2_0) { for (int ax2 = 0; ax2 < 2; ++ax2) { vstore2(((half2)((half)0.000000e+00f, (half)0.000000e+00f)), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + (ax2 * 2))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 4))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 8))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 12))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 16))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 20))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 24))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 28))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); } } for (int ax2_fused_2 = 0; ax2_fused_2 < 2; ++ax2_fused_2) { for (int ax1 = 0; ax1 < 2; ++ax1) { barrier(CLK_LOCAL_MEM_FENCE); ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_intermediate_pad_rf_local_1[((ax1 * 2) + ax2_fused_2)]; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 16) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 8) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 4) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 2) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); } barrier(CLK_LOCAL_MEM_FENCE); NT_matmul_intermediate_pad_local[((ax1 * 2) + ax2_fused_2)] = ((volatile __local half*)red_buf0)[((convert_int(get_local_id(0))) * 32)]; } } for (int ax0 = 0; ax0 < 2; ++ax0) { for (int ax1_fused_2 = 0; ax1_fused_2 < 2; ++ax1_fused_2) { if (((bool)(((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)(((bool)(((convert_int(get_group_id(1))) - ((seq_len + 1) >> 1)) < 0)) || ((bool)((convert_int(get_group_id(1))) == 0)))))) && ((bool)((((convert_int(get_group_id(1))) * 2) + ax0) < seq_len))) { T_add[((((((convert_int(get_group_id(1))) * 12288) + (ax0 * 6144)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] = (NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_attention_query_key_value_bias3[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]); } } } } // Function: gpt_neox_q4f16_1_batch_decode_paged_kv_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode) { int kv_chunk_len[1]; float st_m[1]; float st_d[1]; float4 O_local[1]; half4 Q_local[1]; float m_prev[1]; __local float4 O_allreduce[256]; __local float md_allreduce[16]; __local half4 K_smem[256]; __local half4 V_smem[256]; float QK_local[4]; float S_reduce_local[1]; __local float red_buf0[256]; float S_local[1]; half4 V_local[1]; float d_prev[1]; float other_m[1]; float other_d[1]; float4 other_o[1]; float exp_mprev[1]; int cur_page_indptr_begin = page_table_indptr[((convert_int(get_group_id(0))) + page_indptr_elem_offset)]; int cur_page_indptr_end = page_table_indptr[(((convert_int(get_group_id(0))) + page_indptr_elem_offset) + 1)]; int condval; if ((cur_page_indptr_begin != cur_page_indptr_end)) { condval = ((((cur_page_indptr_end * 16) + length_info[((convert_int(get_group_id(0))) + length_info_elem_offset)]) - (cur_page_indptr_begin * 16)) - 16); } else { condval = 0; } kv_chunk_len[0] = condval; st_m[0] = -5.000000e+04f; st_d[0] = 1.000000e+00f; O_local[0] = ((float4)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)); half4 condval_1; if ((rotary_mode == 1)) { float4 freq = (((float4)(((convert_float(q_rope_position[((convert_int(get_group_id(0))) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[((convert_int(get_group_id(0))) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[((convert_int(get_group_id(0))) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[((convert_int(get_group_id(0))) + q_rope_position_elem_offset)])) * rope_scale))) / pow(((float4)(rope_theta, rope_theta, rope_theta, rope_theta)), ((convert_float4(((int4)(((((convert_int(get_local_id(0))) & 15) * 8))+(2*0), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*1), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*2), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*3))))) * ((float4)(7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f))))); half4 condval_2; if (((convert_int(get_local_id(0))) < 16)) { condval_2 = (vload4(0, Q + (((((convert_int(get_group_id(0))) * 2048) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + 64)) * ((half4)((half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f))); } else { condval_2 = vload4(0, Q + (((((convert_int(get_group_id(0))) * 2048) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) - 64)); } condval_1 = (convert_half4(((cos(freq) * (convert_float4(vload4(0, Q + ((((convert_int(get_group_id(0))) * 2048) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)))))) + (sin(freq) * (convert_float4(condval_2)))))); } else { condval_1 = vload4(0, Q + ((((convert_int(get_group_id(0))) * 2048) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); } Q_local[0] = condval_1; for (int iterator = 0; iterator < ((kv_chunk_len[0] + 7) >> 3); ++iterator) { if (((iterator * 8) + (convert_int(get_local_id(2)))) < kv_chunk_len[0]) { int page_no = page_table_values[(((iterator >> 1) + cur_page_indptr_begin) + page_values_elem_offset)]; half4 condval_3; if ((rotary_mode == 1)) { float4 freq_1 = (((float4)(((convert_float((((iterator * 8) + k_rope_pos_offset[((convert_int(get_group_id(0))) + k_rope_pos_offset_elem_offset)]) + (convert_int(get_local_id(2)))))) * rope_scale), ((convert_float((((iterator * 8) + k_rope_pos_offset[((convert_int(get_group_id(0))) + k_rope_pos_offset_elem_offset)]) + (convert_int(get_local_id(2)))))) * rope_scale), ((convert_float((((iterator * 8) + k_rope_pos_offset[((convert_int(get_group_id(0))) + k_rope_pos_offset_elem_offset)]) + (convert_int(get_local_id(2)))))) * rope_scale), ((convert_float((((iterator * 8) + k_rope_pos_offset[((convert_int(get_group_id(0))) + k_rope_pos_offset_elem_offset)]) + (convert_int(get_local_id(2)))))) * rope_scale))) / pow(((float4)(rope_theta, rope_theta, rope_theta, rope_theta)), ((convert_float4(((int4)(((((convert_int(get_local_id(0))) & 15) * 8))+(2*0), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*1), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*2), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*3))))) * ((float4)(7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f))))); int4 v_ = (int4)((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*0), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*1), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*2), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*3)); half4 condval_4; if (((convert_int(get_local_id(0))) < 16)) { int4 v__1 = (int4)(((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 64))+(1*0), ((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 64))+(1*1), ((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 64))+(1*2), ((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 64))+(1*3)); condval_4 = (((half4)(pages[v__1.s0],pages[v__1.s1],pages[v__1.s2],pages[v__1.s3])) * ((half4)((half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f))); } else { int4 v__2 = (int4)(((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) - 64))+(1*0), ((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) - 64))+(1*1), ((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) - 64))+(1*2), ((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) - 64))+(1*3)); condval_4 = ((half4)(pages[v__2.s0],pages[v__2.s1],pages[v__2.s2],pages[v__2.s3])); } condval_3 = (convert_half4(((cos(freq_1) * (convert_float4(((half4)(pages[v_.s0],pages[v_.s1],pages[v_.s2],pages[v_.s3]))))) + (sin(freq_1) * (convert_float4(condval_4)))))); } else { int4 v__3 = (int4)((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*0), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*1), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*2), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*3)); condval_3 = ((half4)(pages[v__3.s0],pages[v__3.s1],pages[v__3.s2],pages[v__3.s3])); } K_smem[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = condval_3; int4 v__4 = (int4)(((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 32768))+(1*0), ((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 32768))+(1*1), ((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 32768))+(1*2), ((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((iterator & 1) * 1024)) + ((convert_int(get_local_id(2))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 32768))+(1*3)); V_smem[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = ((half4)(pages[v__4.s0],pages[v__4.s1],pages[v__4.s2],pages[v__4.s3])); } else { K_smem[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = ((half4)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)); V_smem[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = ((half4)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)); } barrier(CLK_LOCAL_MEM_FENCE); m_prev[0] = st_m[0]; vstore4(((((convert_float4(Q_local[0])) * (convert_float4(K_smem[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))]))) * ((float4)(attn_score_scaling_factor, attn_score_scaling_factor, attn_score_scaling_factor, attn_score_scaling_factor))) * ((float4)(1.275174e-01f, 1.275174e-01f, 1.275174e-01f, 1.275174e-01f))), 0, QK_local + 0); S_reduce_local[0] = 0.000000e+00f; S_reduce_local[0] = (S_reduce_local[0] + QK_local[0]); S_reduce_local[0] = (S_reduce_local[0] + QK_local[1]); S_reduce_local[0] = (S_reduce_local[0] + QK_local[2]); S_reduce_local[0] = (S_reduce_local[0] + QK_local[3]); barrier(CLK_LOCAL_MEM_FENCE); ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = S_reduce_local[0]; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 16) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0)))) + 16)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 8) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0)))) + 8)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 4) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0)))) + 4)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 2) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0)))) + 2)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 1) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0)))) + 1)]); } barrier(CLK_LOCAL_MEM_FENCE); S_local[0] = -5.000000e+04f; if (((iterator * 8) + (convert_int(get_local_id(2)))) < kv_chunk_len[0]) { S_local[0] = ((volatile __local float*)red_buf0)[((convert_int(get_local_id(2))) * 32)]; } st_m[0] = max((float)st_m[0], (float)S_local[0]); float o_scale = exp2((m_prev[0] - st_m[0])); st_d[0] = (st_d[0] * o_scale); S_local[0] = exp2((S_local[0] - st_m[0])); st_d[0] = (st_d[0] + S_local[0]); O_local[0] = (O_local[0] * ((float4)(o_scale, o_scale, o_scale, o_scale))); V_local[0] = V_smem[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))]; O_local[0] = (O_local[0] + ((convert_float4(V_local[0])) * ((float4)(S_local[0], S_local[0], S_local[0], S_local[0])))); } O_allreduce[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = O_local[0]; md_allreduce[((convert_int(get_local_id(2))) * 2)] = st_m[0]; md_allreduce[(((convert_int(get_local_id(2))) * 2) + 1)] = st_d[0]; barrier(CLK_LOCAL_MEM_FENCE); st_m[0] = -5.000000e+04f; st_d[0] = 1.000000e+00f; O_local[0] = ((float4)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)); for (int j = 0; j < 8; ++j) { m_prev[0] = st_m[0]; d_prev[0] = st_d[0]; other_m[0] = md_allreduce[(j * 2)]; other_d[0] = md_allreduce[((j * 2) + 1)]; other_o[0] = O_allreduce[((j * 32) + (convert_int(get_local_id(0))))]; st_m[0] = max((float)st_m[0], (float)other_m[0]); st_d[0] = ((d_prev[0] * exp2((m_prev[0] - st_m[0]))) + (other_d[0] * exp2((other_m[0] - st_m[0])))); exp_mprev[0] = exp2((m_prev[0] - st_m[0])); other_m[0] = exp2((other_m[0] - st_m[0])); O_local[0] = ((O_local[0] * ((float4)(exp_mprev[0], exp_mprev[0], exp_mprev[0], exp_mprev[0]))) + (other_o[0] * ((float4)(other_m[0], other_m[0], other_m[0], other_m[0])))); } O_local[0] = (O_local[0] / ((float4)(st_d[0], st_d[0], st_d[0], st_d[0]))); vstore4((convert_half4(O_local[0])), 0, output + ((((convert_int(get_group_id(0))) * 2048) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); lse[(((convert_int(get_group_id(0))) * 16) + (convert_int(get_group_id(1))))] = (st_m[0] + log2(st_d[0])); } // Function: gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293) { half NT_matmul_rf_local[8]; uint gpt_neox_layers_0_attention_dense_q_weight2_local[1]; half NT_matmul_rf_local_1[1]; __local half red_buf0[64]; vstore8(((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, NT_matmul_rf_local + 0); for (int ax1_0_fused_ax1_1_fused_0 = 0; ax1_0_fused_ax1_1_fused_0 < 8; ++ax1_0_fused_ax1_1_fused_0) { gpt_neox_layers_0_attention_dense_q_weight2_local[0] = gpt_neox_layers_0_attention_dense_q_weight2[(((((convert_int(get_group_id(0))) * 512) + ((convert_int(get_local_id(0))) * 256)) + (ax1_0_fused_ax1_1_fused_0 * 32)) + (convert_int(get_local_id(1))))]; vstore8((vload8(0, NT_matmul_rf_local + 0) + (vload8(0, lv293 + ((ax1_0_fused_ax1_1_fused_0 * 256) + ((convert_int(get_local_id(1))) * 8))) * (((convert_half8(((((uint8)(gpt_neox_layers_0_attention_dense_q_weight2_local[0], gpt_neox_layers_0_attention_dense_q_weight2_local[0], gpt_neox_layers_0_attention_dense_q_weight2_local[0], gpt_neox_layers_0_attention_dense_q_weight2_local[0], gpt_neox_layers_0_attention_dense_q_weight2_local[0], gpt_neox_layers_0_attention_dense_q_weight2_local[0], gpt_neox_layers_0_attention_dense_q_weight2_local[0], gpt_neox_layers_0_attention_dense_q_weight2_local[0])) >> ((uint8)(((uint)0)+((uint)4*0), ((uint)0)+((uint)4*1), ((uint)0)+((uint)4*2), ((uint)0)+((uint)4*3), ((uint)0)+((uint)4*4), ((uint)0)+((uint)4*5), ((uint)0)+((uint)4*6), ((uint)0)+((uint)4*7)))) & ((uint8)((uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15, (uint)15))))) - ((half8)((half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f, (half)7.000000e+00f))) * ((half8)(gpt_neox_layers_0_attention_dense_q_scale2[(((((convert_int(get_group_id(0))) * 128) + ((convert_int(get_local_id(0))) * 64)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], gpt_neox_layers_0_attention_dense_q_scale2[(((((convert_int(get_group_id(0))) * 128) + ((convert_int(get_local_id(0))) * 64)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], gpt_neox_layers_0_attention_dense_q_scale2[(((((convert_int(get_group_id(0))) * 128) + ((convert_int(get_local_id(0))) * 64)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], gpt_neox_layers_0_attention_dense_q_scale2[(((((convert_int(get_group_id(0))) * 128) + ((convert_int(get_local_id(0))) * 64)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], gpt_neox_layers_0_attention_dense_q_scale2[(((((convert_int(get_group_id(0))) * 128) + ((convert_int(get_local_id(0))) * 64)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], gpt_neox_layers_0_attention_dense_q_scale2[(((((convert_int(get_group_id(0))) * 128) + ((convert_int(get_local_id(0))) * 64)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], gpt_neox_layers_0_attention_dense_q_scale2[(((((convert_int(get_group_id(0))) * 128) + ((convert_int(get_local_id(0))) * 64)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))], gpt_neox_layers_0_attention_dense_q_scale2[(((((convert_int(get_group_id(0))) * 128) + ((convert_int(get_local_id(0))) * 64)) + (ax1_0_fused_ax1_1_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]))))), 0, NT_matmul_rf_local + 0); } for (int ax2_fused_2_0 = 0; ax2_fused_2_0 < 1; ++ax2_fused_2_0) { NT_matmul_rf_local_1[0] = (half)0.000000e+00f; NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[0]); NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[1]); NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[2]); NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[3]); NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[4]); NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[5]); NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[6]); NT_matmul_rf_local_1[0] = (NT_matmul_rf_local_1[0] + NT_matmul_rf_local[7]); } barrier(CLK_LOCAL_MEM_FENCE); ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_rf_local_1[0]; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 16) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 8) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 4) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 2) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) == 0) { T_add[(((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0))))] = ((lv218[(((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0))))] + (((volatile __local half*)red_buf0)[((convert_int(get_local_id(0))) * 32)] + gpt_neox_layers_0_attention_dense_bias2[(((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0))))])) + input_embed[(((convert_int(get_group_id(0))) * 2) + (convert_int(get_local_id(0))))]); } } // Function: gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size) { half NT_matmul_intermediate_reindex_pad_local[16]; for (int var = 0; var < 1; ++var) { vstore8(((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, NT_matmul_intermediate_reindex_pad_local + 0); vstore8(((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, NT_matmul_intermediate_reindex_pad_local + 8); for (int ax3_0 = 0; ax3_0 < 128; ++ax3_0) { for (int ax3_1 = 0; ax3_1 < 16; ++ax3_1) { uint cse_var_1 = (convert_uint(((ax3_1 & 7) * 4))); half condval; if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { condval = layer_norm195[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; } else { condval = (half)0.000000e+00f; } half condval_1; if (((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) < vocab_size)) { condval_1 = (((convert_half(((embed_out_q_weight4[(((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3))] >> cse_var_1) & (uint)15))) - (half)7.000000e+00f) * embed_out_q_scale4[((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1))]); } else { condval_1 = (half)0.000000e+00f; } NT_matmul_intermediate_reindex_pad_local[0] = (NT_matmul_intermediate_reindex_pad_local[0] + (condval * condval_1)); half condval_2; if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { condval_2 = layer_norm195[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; } else { condval_2 = (half)0.000000e+00f; } half condval_3; if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 1) < vocab_size)) { condval_3 = (((convert_half(((embed_out_q_weight4[((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)) + 256)] >> cse_var_1) & (uint)15))) - (half)7.000000e+00f) * embed_out_q_scale4[(((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)) + 64)]); } else { condval_3 = (half)0.000000e+00f; } NT_matmul_intermediate_reindex_pad_local[1] = (NT_matmul_intermediate_reindex_pad_local[1] + (condval_2 * condval_3)); half condval_4; if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { condval_4 = layer_norm195[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; } else { condval_4 = (half)0.000000e+00f; } half condval_5; if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 2) < vocab_size)) { condval_5 = (((convert_half(((embed_out_q_weight4[((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)) + 512)] >> cse_var_1) & (uint)15))) - (half)7.000000e+00f) * embed_out_q_scale4[(((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)) + 128)]); } else { condval_5 = (half)0.000000e+00f; } NT_matmul_intermediate_reindex_pad_local[2] = (NT_matmul_intermediate_reindex_pad_local[2] + (condval_4 * condval_5)); half condval_6; if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { condval_6 = layer_norm195[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; } else { condval_6 = (half)0.000000e+00f; } half condval_7; if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 3) < vocab_size)) { condval_7 = (((convert_half(((embed_out_q_weight4[((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)) + 768)] >> cse_var_1) & (uint)15))) - (half)7.000000e+00f) * embed_out_q_scale4[(((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)) + 192)]); } else { condval_7 = (half)0.000000e+00f; } NT_matmul_intermediate_reindex_pad_local[3] = (NT_matmul_intermediate_reindex_pad_local[3] + (condval_6 * condval_7)); half condval_8; if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { condval_8 = layer_norm195[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; } else { condval_8 = (half)0.000000e+00f; } half condval_9; if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 4) < vocab_size)) { condval_9 = (((convert_half(((embed_out_q_weight4[((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)) + 1024)] >> cse_var_1) & (uint)15))) - (half)7.000000e+00f) * embed_out_q_scale4[(((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)) + 256)]); } else { condval_9 = (half)0.000000e+00f; } NT_matmul_intermediate_reindex_pad_local[4] = (NT_matmul_intermediate_reindex_pad_local[4] + (condval_8 * condval_9)); half condval_10; if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { condval_10 = layer_norm195[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; } else { condval_10 = (half)0.000000e+00f; } half condval_11; if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 5) < vocab_size)) { condval_11 = (((convert_half(((embed_out_q_weight4[((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)) + 1280)] >> cse_var_1) & (uint)15))) - (half)7.000000e+00f) * embed_out_q_scale4[(((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)) + 320)]); } else { condval_11 = (half)0.000000e+00f; } NT_matmul_intermediate_reindex_pad_local[5] = (NT_matmul_intermediate_reindex_pad_local[5] + (condval_10 * condval_11)); half condval_12; if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { condval_12 = layer_norm195[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; } else { condval_12 = (half)0.000000e+00f; } half condval_13; if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 6) < vocab_size)) { condval_13 = (((convert_half(((embed_out_q_weight4[((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)) + 1536)] >> cse_var_1) & (uint)15))) - (half)7.000000e+00f) * embed_out_q_scale4[(((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)) + 384)]); } else { condval_13 = (half)0.000000e+00f; } NT_matmul_intermediate_reindex_pad_local[6] = (NT_matmul_intermediate_reindex_pad_local[6] + (condval_12 * condval_13)); half condval_14; if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { condval_14 = layer_norm195[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; } else { condval_14 = (half)0.000000e+00f; } half condval_15; if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 7) < vocab_size)) { condval_15 = (((convert_half(((embed_out_q_weight4[((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)) + 1792)] >> cse_var_1) & (uint)15))) - (half)7.000000e+00f) * embed_out_q_scale4[(((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)) + 448)]); } else { condval_15 = (half)0.000000e+00f; } NT_matmul_intermediate_reindex_pad_local[7] = (NT_matmul_intermediate_reindex_pad_local[7] + (condval_14 * condval_15)); half condval_16; if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { condval_16 = layer_norm195[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; } else { condval_16 = (half)0.000000e+00f; } half condval_17; if (((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) < vocab_size)) { condval_17 = (((convert_half(((embed_out_q_weight4[(((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3))] >> cse_var_1) & (uint)15))) - (half)7.000000e+00f) * embed_out_q_scale4[((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1))]); } else { condval_17 = (half)0.000000e+00f; } NT_matmul_intermediate_reindex_pad_local[8] = (NT_matmul_intermediate_reindex_pad_local[8] + (condval_16 * condval_17)); half condval_18; if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { condval_18 = layer_norm195[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; } else { condval_18 = (half)0.000000e+00f; } half condval_19; if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 1) < vocab_size)) { condval_19 = (((convert_half(((embed_out_q_weight4[((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)) + 256)] >> cse_var_1) & (uint)15))) - (half)7.000000e+00f) * embed_out_q_scale4[(((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)) + 64)]); } else { condval_19 = (half)0.000000e+00f; } NT_matmul_intermediate_reindex_pad_local[9] = (NT_matmul_intermediate_reindex_pad_local[9] + (condval_18 * condval_19)); half condval_20; if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { condval_20 = layer_norm195[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; } else { condval_20 = (half)0.000000e+00f; } half condval_21; if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 2) < vocab_size)) { condval_21 = (((convert_half(((embed_out_q_weight4[((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)) + 512)] >> cse_var_1) & (uint)15))) - (half)7.000000e+00f) * embed_out_q_scale4[(((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)) + 128)]); } else { condval_21 = (half)0.000000e+00f; } NT_matmul_intermediate_reindex_pad_local[10] = (NT_matmul_intermediate_reindex_pad_local[10] + (condval_20 * condval_21)); half condval_22; if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { condval_22 = layer_norm195[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; } else { condval_22 = (half)0.000000e+00f; } half condval_23; if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 3) < vocab_size)) { condval_23 = (((convert_half(((embed_out_q_weight4[((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)) + 768)] >> cse_var_1) & (uint)15))) - (half)7.000000e+00f) * embed_out_q_scale4[(((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)) + 192)]); } else { condval_23 = (half)0.000000e+00f; } NT_matmul_intermediate_reindex_pad_local[11] = (NT_matmul_intermediate_reindex_pad_local[11] + (condval_22 * condval_23)); half condval_24; if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { condval_24 = layer_norm195[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; } else { condval_24 = (half)0.000000e+00f; } half condval_25; if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 4) < vocab_size)) { condval_25 = (((convert_half(((embed_out_q_weight4[((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)) + 1024)] >> cse_var_1) & (uint)15))) - (half)7.000000e+00f) * embed_out_q_scale4[(((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)) + 256)]); } else { condval_25 = (half)0.000000e+00f; } NT_matmul_intermediate_reindex_pad_local[12] = (NT_matmul_intermediate_reindex_pad_local[12] + (condval_24 * condval_25)); half condval_26; if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { condval_26 = layer_norm195[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; } else { condval_26 = (half)0.000000e+00f; } half condval_27; if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 5) < vocab_size)) { condval_27 = (((convert_half(((embed_out_q_weight4[((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)) + 1280)] >> cse_var_1) & (uint)15))) - (half)7.000000e+00f) * embed_out_q_scale4[(((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)) + 320)]); } else { condval_27 = (half)0.000000e+00f; } NT_matmul_intermediate_reindex_pad_local[13] = (NT_matmul_intermediate_reindex_pad_local[13] + (condval_26 * condval_27)); half condval_28; if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { condval_28 = layer_norm195[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; } else { condval_28 = (half)0.000000e+00f; } half condval_29; if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 6) < vocab_size)) { condval_29 = (((convert_half(((embed_out_q_weight4[((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)) + 1536)] >> cse_var_1) & (uint)15))) - (half)7.000000e+00f) * embed_out_q_scale4[(((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)) + 384)]); } else { condval_29 = (half)0.000000e+00f; } NT_matmul_intermediate_reindex_pad_local[14] = (NT_matmul_intermediate_reindex_pad_local[14] + (condval_28 * condval_29)); half condval_30; if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { condval_30 = layer_norm195[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; } else { condval_30 = (half)0.000000e+00f; } half condval_31; if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 7) < vocab_size)) { condval_31 = (((convert_half(((embed_out_q_weight4[((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)) + 1792)] >> cse_var_1) & (uint)15))) - (half)7.000000e+00f) * embed_out_q_scale4[(((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)) + 448)]); } else { condval_31 = (half)0.000000e+00f; } NT_matmul_intermediate_reindex_pad_local[15] = (NT_matmul_intermediate_reindex_pad_local[15] + (condval_30 * condval_31)); } } if (((bool)((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) && ((bool)((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) < vocab_size))) { compute[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) * vocab_size))] = (convert_float(NT_matmul_intermediate_reindex_pad_local[0])); } if (((bool)((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 1) < vocab_size))) { compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) * vocab_size)) + 1)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[1])); } if (((bool)((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 2) < vocab_size))) { compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) * vocab_size)) + 2)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[2])); } if (((bool)((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 3) < vocab_size))) { compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) * vocab_size)) + 3)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[3])); } if (((bool)((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 4) < vocab_size))) { compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) * vocab_size)) + 4)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[4])); } if (((bool)((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 5) < vocab_size))) { compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) * vocab_size)) + 5)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[5])); } if (((bool)((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 6) < vocab_size))) { compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) * vocab_size)) + 6)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[6])); } if (((bool)((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 7) < vocab_size))) { compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) * vocab_size)) + 7)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[7])); } if (((bool)(((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) && ((bool)((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) < vocab_size))) { compute[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) * vocab_size))] = (convert_float(NT_matmul_intermediate_reindex_pad_local[8])); } if (((bool)(((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 1) < vocab_size))) { compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) * vocab_size)) + 1)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[9])); } if (((bool)(((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 2) < vocab_size))) { compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) * vocab_size)) + 2)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[10])); } if (((bool)(((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 3) < vocab_size))) { compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) * vocab_size)) + 3)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[11])); } if (((bool)(((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 4) < vocab_size))) { compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) * vocab_size)) + 4)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[12])); } if (((bool)(((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 5) < vocab_size))) { compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) * vocab_size)) + 5)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[13])); } if (((bool)(((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 6) < vocab_size))) { compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) * vocab_size)) + 6)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[14])); } if (((bool)(((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 7) < vocab_size))) { compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) * vocab_size)) + 7)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[15])); } } } // Function: gpt_neox_q4f16_1_reshape1_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size) { T_reshape[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))] = reshape288[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]; } // Function: gpt_neox_q4f16_1_batch_tree_attn_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size) { int tile_id[1]; int batch_idx[1]; int batch_rows[1]; int batch_tiles[1]; int kv_chunk_len[1]; __local float m_smem[32]; __local float d_smem[32]; float O_local[32]; __local half Q_smem[4096]; __local half K_smem[4096]; __local half V_smem[4096]; float S_local[8]; __local float S_smem[1024]; float m_prev[1]; float m_new[1]; float d_new[1]; __local float m_prev_smem[32]; tile_id[0] = (convert_int(get_group_id(0))); batch_idx[0] = 0; batch_rows[0] = (q_indptr[(q_indptr_elem_offset + 1)] - q_indptr[q_indptr_elem_offset]); batch_tiles[0] = ((batch_rows[0] + 31) >> 5); while (1) { if (!(((batch_idx[0] < batch_size)))) { break; } while (1) { if (!((((bool)(batch_tiles[0] <= tile_id[0])) && ((bool)(batch_idx[0] < batch_size))))) { break; } tile_id[0] = (tile_id[0] - batch_tiles[0]); batch_idx[0] = (batch_idx[0] + 1); if (batch_idx[0] < batch_size) { int b_idx = batch_idx[0]; batch_rows[0] = (q_indptr[((b_idx + q_indptr_elem_offset) + 1)] - q_indptr[(b_idx + q_indptr_elem_offset)]); batch_tiles[0] = ((batch_rows[0] + 31) >> 5); } } if ((batch_idx[0] < batch_size)) { int b_idx_1 = batch_idx[0]; int LH_start = (tile_id[0] * 32); int q_indptr_val = q_indptr[(b_idx_1 + q_indptr_elem_offset)]; kv_chunk_len[0] = (kv_indptr[((b_idx_1 + kv_indptr_elem_offset) + 1)] - kv_indptr[(b_idx_1 + kv_indptr_elem_offset)]); barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { m_smem[(convert_int(get_local_id(0)))] = -5.000000e+04f; d_smem[(convert_int(get_local_id(0)))] = 1.000000e+00f; } for (int li_1 = 0; li_1 < 4; ++li_1) { for (int lj_1 = 0; lj_1 < 8; ++lj_1) { O_local[((li_1 * 8) + lj_1)] = 0.000000e+00f; } } barrier(CLK_LOCAL_MEM_FENCE); for (int li_lj_fused_0 = 0; li_lj_fused_0 < 8; ++li_lj_fused_0) { if (((((li_lj_fused_0 * 4) + q_indptr_val) + LH_start) + (convert_int(get_local_id(1)))) < q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)]) { half4 condval; if ((rotary_mode == 1)) { float4 freq = (((float4)(((convert_float(q_rope_position[(((((li_lj_fused_0 * 4) + (convert_int(get_local_id(1)))) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[(((((li_lj_fused_0 * 4) + (convert_int(get_local_id(1)))) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[(((((li_lj_fused_0 * 4) + (convert_int(get_local_id(1)))) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[(((((li_lj_fused_0 * 4) + (convert_int(get_local_id(1)))) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale))) / pow(((float4)(rope_theta, rope_theta, rope_theta, rope_theta)), ((convert_float4(((int4)(((((convert_int(get_local_id(0))) & 15) * 8))+(2*0), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*1), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*2), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*3))))) * ((float4)(7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f))))); half4 condval_1; if (((convert_int(get_local_id(0))) < 16)) { condval_1 = (vload4(0, q + (((((((li_lj_fused_0 * 8192) + ((convert_int(get_local_id(1))) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + 64)) * ((half4)((half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f))); } else { condval_1 = vload4(0, q + (((((((li_lj_fused_0 * 8192) + ((convert_int(get_local_id(1))) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) - 64)); } condval = (convert_half4(((cos(freq) * (convert_float4(vload4(0, q + ((((((li_lj_fused_0 * 8192) + ((convert_int(get_local_id(1))) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)))))) + (sin(freq) * (convert_float4(condval_1)))))); } else { condval = vload4(0, q + ((((((li_lj_fused_0 * 8192) + ((convert_int(get_local_id(1))) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); } vstore4(condval, 0, Q_smem + (((li_lj_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); } else { vstore4(((half4)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, Q_smem + (((li_lj_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); } } barrier(CLK_LOCAL_MEM_FENCE); for (int iterator = 0; iterator < ((kv_chunk_len[0] + 31) >> 5); ++iterator) { int L_kv_base = kv_indptr[(b_idx_1 + kv_indptr_elem_offset)]; barrier(CLK_LOCAL_MEM_FENCE); for (int lz_ly_fused_0 = 0; lz_ly_fused_0 < 8; ++lz_ly_fused_0) { if ((((iterator * 32) + (lz_ly_fused_0 * 4)) + (convert_int(get_local_id(1)))) < kv_chunk_len[0]) { half4 condval_2; if ((rotary_mode == 1)) { float4 freq_1 = (((float4)(((convert_float(q_rope_position[(((((iterator * 32) + (lz_ly_fused_0 * 4)) + (convert_int(get_local_id(1)))) + L_kv_base) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[(((((iterator * 32) + (lz_ly_fused_0 * 4)) + (convert_int(get_local_id(1)))) + L_kv_base) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[(((((iterator * 32) + (lz_ly_fused_0 * 4)) + (convert_int(get_local_id(1)))) + L_kv_base) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[(((((iterator * 32) + (lz_ly_fused_0 * 4)) + (convert_int(get_local_id(1)))) + L_kv_base) + q_rope_position_elem_offset)])) * rope_scale))) / pow(((float4)(rope_theta, rope_theta, rope_theta, rope_theta)), ((convert_float4(((int4)(((((convert_int(get_local_id(0))) & 15) * 8))+(2*0), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*1), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*2), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*3))))) * ((float4)(7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f))))); half4 condval_3; if (((convert_int(get_local_id(0))) < 16)) { condval_3 = (vload4(0, k + (((((((iterator * 65536) + (lz_ly_fused_0 * 8192)) + ((convert_int(get_local_id(1))) * 2048)) + (L_kv_base * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + 64)) * ((half4)((half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f))); } else { condval_3 = vload4(0, k + (((((((iterator * 65536) + (lz_ly_fused_0 * 8192)) + ((convert_int(get_local_id(1))) * 2048)) + (L_kv_base * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) - 64)); } condval_2 = (convert_half4(((cos(freq_1) * (convert_float4(vload4(0, k + ((((((iterator * 65536) + (lz_ly_fused_0 * 8192)) + ((convert_int(get_local_id(1))) * 2048)) + (L_kv_base * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)))))) + (sin(freq_1) * (convert_float4(condval_3)))))); } else { condval_2 = vload4(0, k + ((((((iterator * 65536) + (lz_ly_fused_0 * 8192)) + ((convert_int(get_local_id(1))) * 2048)) + (L_kv_base * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); } vstore4(condval_2, 0, K_smem + (((lz_ly_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); vstore4(vload4(0, v + ((((((iterator * 65536) + (lz_ly_fused_0 * 8192)) + ((convert_int(get_local_id(1))) * 2048)) + (L_kv_base * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))), 0, V_smem + (((lz_ly_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); } else { vstore4(((half4)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, K_smem + (((lz_ly_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); vstore4(((half4)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, V_smem + (((lz_ly_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); } } barrier(CLK_LOCAL_MEM_FENCE); for (int li_1_init = 0; li_1_init < 2; ++li_1_init) { for (int lj_1_init = 0; lj_1_init < 4; ++lj_1_init) { S_local[((li_1_init * 4) + lj_1_init)] = 0.000000e+00f; } } for (int lk_0 = 0; lk_0 < 16; ++lk_0) { for (int li_1_1 = 0; li_1_1 < 2; ++li_1_1) { for (int lj_1_1 = 0; lj_1_1 < 4; ++lj_1_1) { for (int lk_1 = 0; lk_1 < 8; ++lk_1) { S_local[((li_1_1 * 4) + lj_1_1)] = (S_local[((li_1_1 * 4) + lj_1_1)] + ((((convert_float(Q_smem[((((((convert_int(get_local_id(1))) * 1024) + (((convert_int(get_local_id(0))) >> 3) * 256)) + (li_1_1 * 128)) + (lk_0 * 8)) + lk_1)])) * (convert_float(K_smem[((((((convert_int(get_local_id(0))) & 7) * 512) + (lj_1_1 * 128)) + (lk_0 * 8)) + lk_1)]))) * attn_score_scaling_factor) * 1.275174e-01f)); } } } } barrier(CLK_LOCAL_MEM_FENCE); for (int li_1_2 = 0; li_1_2 < 2; ++li_1_2) { for (int lj_1_2 = 0; lj_1_2 < 4; ++lj_1_2) { S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 3) * 64)) + (li_1_2 * 32)) + (((convert_int(get_local_id(0))) & 7) * 4)) + lj_1_2)] = S_local[((li_1_2 * 4) + lj_1_2)]; } } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { m_prev[0] = m_smem[(convert_int(get_local_id(0)))]; m_new[0] = m_smem[(convert_int(get_local_id(0)))]; for (int j = 0; j < 32; ++j) { int cse_var_4 = ((iterator * 32) + j); int cse_var_3 = ((iterator * 64) + (j * 2)); if (((bool)(cse_var_4 < kv_chunk_len[0])) && ((bool)(((bool)(cse_var_4 < ((kv_chunk_len[0] + mn_indptr[(b_idx_1 + mn_indptr_elem_offset)]) - mn_indptr[((b_idx_1 + mn_indptr_elem_offset) + 1)]))) || ((bool)(((bool)(mask[((((mn_indptr[((b_idx_1 + mn_indptr_elem_offset) + 1)] * 2) + cse_var_3) + mask_elem_offset) - (kv_chunk_len[0] * 2))] <= mask[(((((((convert_int(get_local_id(0))) * 2) + (LH_start * 2)) + (mn_indptr[((b_idx_1 + mn_indptr_elem_offset) + 1)] * 2)) + (q_indptr[(b_idx_1 + q_indptr_elem_offset)] * 2)) + mask_elem_offset) - (q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)] * 2))])) && ((bool)(mask[(((((((convert_int(get_local_id(0))) * 2) + (LH_start * 2)) + (mn_indptr[((b_idx_1 + mn_indptr_elem_offset) + 1)] * 2)) + (q_indptr[(b_idx_1 + q_indptr_elem_offset)] * 2)) + mask_elem_offset) - (q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)] * 2))] < mask[(((((mn_indptr[((b_idx_1 + mn_indptr_elem_offset) + 1)] * 2) + cse_var_3) + mask_elem_offset) + 1) - (kv_chunk_len[0] * 2))]))))))) { m_new[0] = max((float)m_new[0], (float)S_smem[((((convert_int(get_local_id(1))) * 1024) + ((convert_int(get_local_id(0))) * 32)) + j)]); } } d_new[0] = (d_smem[(convert_int(get_local_id(0)))] * exp2((m_prev[0] - m_new[0]))); } for (int j_1 = 0; j_1 < 32; ++j_1) { barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { int cse_var_6 = ((iterator * 32) + j_1); int cse_var_5 = ((iterator * 64) + (j_1 * 2)); if (((bool)(cse_var_6 < kv_chunk_len[0])) && ((bool)(((bool)(cse_var_6 < ((kv_chunk_len[0] + mn_indptr[(b_idx_1 + mn_indptr_elem_offset)]) - mn_indptr[((b_idx_1 + mn_indptr_elem_offset) + 1)]))) || ((bool)(((bool)(mask[((((mn_indptr[((b_idx_1 + mn_indptr_elem_offset) + 1)] * 2) + cse_var_5) + mask_elem_offset) - (kv_chunk_len[0] * 2))] <= mask[(((((((convert_int(get_local_id(0))) * 2) + (LH_start * 2)) + (mn_indptr[((b_idx_1 + mn_indptr_elem_offset) + 1)] * 2)) + (q_indptr[(b_idx_1 + q_indptr_elem_offset)] * 2)) + mask_elem_offset) - (q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)] * 2))])) && ((bool)(mask[(((((((convert_int(get_local_id(0))) * 2) + (LH_start * 2)) + (mn_indptr[((b_idx_1 + mn_indptr_elem_offset) + 1)] * 2)) + (q_indptr[(b_idx_1 + q_indptr_elem_offset)] * 2)) + mask_elem_offset) - (q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)] * 2))] < mask[(((((mn_indptr[((b_idx_1 + mn_indptr_elem_offset) + 1)] * 2) + cse_var_5) + mask_elem_offset) + 1) - (kv_chunk_len[0] * 2))]))))))) { S_smem[((((convert_int(get_local_id(1))) * 1024) + ((convert_int(get_local_id(0))) * 32)) + j_1)] = exp2((S_smem[((((convert_int(get_local_id(1))) * 1024) + ((convert_int(get_local_id(0))) * 32)) + j_1)] - m_new[0])); } else { S_smem[(((convert_int(get_local_id(0))) * 32) + j_1)] = exp2((-5.000000e+04f - m_new[0])); } } } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { for (int j_2 = 0; j_2 < 32; ++j_2) { d_new[0] = (d_new[0] + S_smem[(((convert_int(get_local_id(0))) * 32) + j_2)]); } m_smem[(convert_int(get_local_id(0)))] = m_new[0]; d_smem[(convert_int(get_local_id(0)))] = d_new[0]; m_prev_smem[(convert_int(get_local_id(0)))] = m_prev[0]; } barrier(CLK_LOCAL_MEM_FENCE); for (int li_1_init_1 = 0; li_1_init_1 < 4; ++li_1_init_1) { for (int lj_1_init_1 = 0; lj_1_init_1 < 8; ++lj_1_init_1) { O_local[((li_1_init_1 * 8) + lj_1_init_1)] = (O_local[((li_1_init_1 * 8) + lj_1_init_1)] * exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_init_1)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_init_1)]))); } } for (int lk_0_1 = 0; lk_0_1 < 4; ++lk_0_1) { for (int lk_1_1 = 0; lk_1_1 < 8; ++lk_1_1) { for (int li_1_3 = 0; li_1_3 < 4; ++li_1_3) { for (int lj_1_3 = 0; lj_1_3 < 8; ++lj_1_3) { O_local[((li_1_3 * 8) + lj_1_3)] = (O_local[((li_1_3 * 8) + lj_1_3)] + (S_smem[((((((convert_int(get_local_id(1))) * 256) + (((convert_int(get_local_id(0))) >> 4) * 128)) + (li_1_3 * 32)) + (lk_0_1 * 8)) + lk_1_1)] * (convert_float(V_smem[((((lk_0_1 * 1024) + (lk_1_1 * 128)) + (((convert_int(get_local_id(0))) & 15) * 8)) + lj_1_3)])))); } } } } } for (int li_1_4 = 0; li_1_4 < 4; ++li_1_4) { for (int lj_1_4 = 0; lj_1_4 < 8; ++lj_1_4) { int cur_L = ((((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + q_indptr[(b_idx_1 + q_indptr_elem_offset)]) + LH_start) + li_1_4); if (cur_L < q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)]) { output[((((cur_L * 2048) + ((convert_int(get_group_id(1))) * 128)) + (((convert_int(get_local_id(0))) & 15) * 8)) + lj_1_4)] = (convert_half((O_local[((li_1_4 * 8) + lj_1_4)] / d_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_4)]))); } } } if ((convert_int(get_local_id(1))) < 1) { int cur_L_1 = (((((convert_int(get_local_id(1))) * 32) + q_indptr[(b_idx_1 + q_indptr_elem_offset)]) + LH_start) + (convert_int(get_local_id(0)))); if (cur_L_1 < q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)]) { lse[((cur_L_1 * 16) + (convert_int(get_group_id(1))))] = (m_smem[(convert_int(get_local_id(0)))] + log2(d_smem[(convert_int(get_local_id(0)))])); } } tile_id[0] = (tile_id[0] + 16); } } } // Function: gpt_neox_q4f16_1_merge_state_inplace_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N) { float s_val[1]; float s_other_val[1]; float s_max[1]; float scale[1]; float other_scale[1]; half v_vec[4]; half v_other_vec[4]; s_val[0] = S[((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_group_id(0))) * H)) + (convert_int(get_local_id(1))))]; s_other_val[0] = S_other[((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_group_id(0))) * H)) + (convert_int(get_local_id(1))))]; s_max[0] = max((float)s_val[0], (float)s_other_val[0]); s_val[0] = exp2((s_val[0] - s_max[0])); s_other_val[0] = exp2((s_other_val[0] - s_max[0])); scale[0] = (s_val[0] / (s_val[0] + s_other_val[0])); other_scale[0] = (s_other_val[0] / (s_val[0] + s_other_val[0])); int4 v_ = (int4)(((((convert_int(get_local_id(0))) * 4) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_group_id(0))) * H)) + (convert_int(get_local_id(1)))) * D)))+(1*0), ((((convert_int(get_local_id(0))) * 4) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_group_id(0))) * H)) + (convert_int(get_local_id(1)))) * D)))+(1*1), ((((convert_int(get_local_id(0))) * 4) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_group_id(0))) * H)) + (convert_int(get_local_id(1)))) * D)))+(1*2), ((((convert_int(get_local_id(0))) * 4) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_group_id(0))) * H)) + (convert_int(get_local_id(1)))) * D)))+(1*3)); vstore4(((half4)(V[v_.s0],V[v_.s1],V[v_.s2],V[v_.s3])), 0, v_vec + 0); vstore4(((half4)(V_other[v_.s0],V_other[v_.s1],V_other[v_.s2],V_other[v_.s3])), 0, v_other_vec + 0); for (int vec = 0; vec < 4; ++vec) { v_vec[vec] = (convert_half((((convert_float(v_vec[vec])) * scale[0]) + ((convert_float(v_other_vec[vec])) * other_scale[0])))); } vstore4(vload4(0, v_vec + 0), 0, V + (((convert_int(get_local_id(0))) * 4) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_group_id(0))) * H)) + (convert_int(get_local_id(1)))) * D))); S[((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_group_id(0))) * H)) + (convert_int(get_local_id(1))))] = (log2((s_val[0] + s_other_val[0])) + s_max[0]); } // Function: gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size) { half NT_matmul_intermediate_reindex_pad_local[16]; for (int var = 0; var < 1; ++var) { vstore8(((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, NT_matmul_intermediate_reindex_pad_local + 0); vstore8(((half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, NT_matmul_intermediate_reindex_pad_local + 8); for (int ax3_0 = 0; ax3_0 < 128; ++ax3_0) { for (int ax3_1 = 0; ax3_1 < 16; ++ax3_1) { uint cse_var_1 = (convert_uint(((ax3_1 & 7) * 4))); half condval; if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { condval = take1[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; } else { condval = (half)0.000000e+00f; } half condval_1; if (((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) < vocab_size)) { condval_1 = (((convert_half(((embed_out_q_weight3[(((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3))] >> cse_var_1) & (uint)15))) - (half)7.000000e+00f) * embed_out_q_scale3[((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1))]); } else { condval_1 = (half)0.000000e+00f; } NT_matmul_intermediate_reindex_pad_local[0] = (NT_matmul_intermediate_reindex_pad_local[0] + (condval * condval_1)); half condval_2; if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { condval_2 = take1[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; } else { condval_2 = (half)0.000000e+00f; } half condval_3; if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 1) < vocab_size)) { condval_3 = (((convert_half(((embed_out_q_weight3[((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)) + 256)] >> cse_var_1) & (uint)15))) - (half)7.000000e+00f) * embed_out_q_scale3[(((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)) + 64)]); } else { condval_3 = (half)0.000000e+00f; } NT_matmul_intermediate_reindex_pad_local[1] = (NT_matmul_intermediate_reindex_pad_local[1] + (condval_2 * condval_3)); half condval_4; if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { condval_4 = take1[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; } else { condval_4 = (half)0.000000e+00f; } half condval_5; if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 2) < vocab_size)) { condval_5 = (((convert_half(((embed_out_q_weight3[((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)) + 512)] >> cse_var_1) & (uint)15))) - (half)7.000000e+00f) * embed_out_q_scale3[(((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)) + 128)]); } else { condval_5 = (half)0.000000e+00f; } NT_matmul_intermediate_reindex_pad_local[2] = (NT_matmul_intermediate_reindex_pad_local[2] + (condval_4 * condval_5)); half condval_6; if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { condval_6 = take1[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; } else { condval_6 = (half)0.000000e+00f; } half condval_7; if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 3) < vocab_size)) { condval_7 = (((convert_half(((embed_out_q_weight3[((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)) + 768)] >> cse_var_1) & (uint)15))) - (half)7.000000e+00f) * embed_out_q_scale3[(((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)) + 192)]); } else { condval_7 = (half)0.000000e+00f; } NT_matmul_intermediate_reindex_pad_local[3] = (NT_matmul_intermediate_reindex_pad_local[3] + (condval_6 * condval_7)); half condval_8; if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { condval_8 = take1[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; } else { condval_8 = (half)0.000000e+00f; } half condval_9; if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 4) < vocab_size)) { condval_9 = (((convert_half(((embed_out_q_weight3[((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)) + 1024)] >> cse_var_1) & (uint)15))) - (half)7.000000e+00f) * embed_out_q_scale3[(((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)) + 256)]); } else { condval_9 = (half)0.000000e+00f; } NT_matmul_intermediate_reindex_pad_local[4] = (NT_matmul_intermediate_reindex_pad_local[4] + (condval_8 * condval_9)); half condval_10; if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { condval_10 = take1[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; } else { condval_10 = (half)0.000000e+00f; } half condval_11; if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 5) < vocab_size)) { condval_11 = (((convert_half(((embed_out_q_weight3[((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)) + 1280)] >> cse_var_1) & (uint)15))) - (half)7.000000e+00f) * embed_out_q_scale3[(((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)) + 320)]); } else { condval_11 = (half)0.000000e+00f; } NT_matmul_intermediate_reindex_pad_local[5] = (NT_matmul_intermediate_reindex_pad_local[5] + (condval_10 * condval_11)); half condval_12; if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { condval_12 = take1[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; } else { condval_12 = (half)0.000000e+00f; } half condval_13; if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 6) < vocab_size)) { condval_13 = (((convert_half(((embed_out_q_weight3[((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)) + 1536)] >> cse_var_1) & (uint)15))) - (half)7.000000e+00f) * embed_out_q_scale3[(((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)) + 384)]); } else { condval_13 = (half)0.000000e+00f; } NT_matmul_intermediate_reindex_pad_local[6] = (NT_matmul_intermediate_reindex_pad_local[6] + (condval_12 * condval_13)); half condval_14; if (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) { condval_14 = take1[(((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1)]; } else { condval_14 = (half)0.000000e+00f; } half condval_15; if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 7) < vocab_size)) { condval_15 = (((convert_half(((embed_out_q_weight3[((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)) + 1792)] >> cse_var_1) & (uint)15))) - (half)7.000000e+00f) * embed_out_q_scale3[(((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)) + 448)]); } else { condval_15 = (half)0.000000e+00f; } NT_matmul_intermediate_reindex_pad_local[7] = (NT_matmul_intermediate_reindex_pad_local[7] + (condval_14 * condval_15)); half condval_16; if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { condval_16 = take1[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; } else { condval_16 = (half)0.000000e+00f; } half condval_17; if (((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) < vocab_size)) { condval_17 = (((convert_half(((embed_out_q_weight3[(((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3))] >> cse_var_1) & (uint)15))) - (half)7.000000e+00f) * embed_out_q_scale3[((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1))]); } else { condval_17 = (half)0.000000e+00f; } NT_matmul_intermediate_reindex_pad_local[8] = (NT_matmul_intermediate_reindex_pad_local[8] + (condval_16 * condval_17)); half condval_18; if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { condval_18 = take1[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; } else { condval_18 = (half)0.000000e+00f; } half condval_19; if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 1) < vocab_size)) { condval_19 = (((convert_half(((embed_out_q_weight3[((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)) + 256)] >> cse_var_1) & (uint)15))) - (half)7.000000e+00f) * embed_out_q_scale3[(((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)) + 64)]); } else { condval_19 = (half)0.000000e+00f; } NT_matmul_intermediate_reindex_pad_local[9] = (NT_matmul_intermediate_reindex_pad_local[9] + (condval_18 * condval_19)); half condval_20; if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { condval_20 = take1[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; } else { condval_20 = (half)0.000000e+00f; } half condval_21; if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 2) < vocab_size)) { condval_21 = (((convert_half(((embed_out_q_weight3[((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)) + 512)] >> cse_var_1) & (uint)15))) - (half)7.000000e+00f) * embed_out_q_scale3[(((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)) + 128)]); } else { condval_21 = (half)0.000000e+00f; } NT_matmul_intermediate_reindex_pad_local[10] = (NT_matmul_intermediate_reindex_pad_local[10] + (condval_20 * condval_21)); half condval_22; if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { condval_22 = take1[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; } else { condval_22 = (half)0.000000e+00f; } half condval_23; if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 3) < vocab_size)) { condval_23 = (((convert_half(((embed_out_q_weight3[((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)) + 768)] >> cse_var_1) & (uint)15))) - (half)7.000000e+00f) * embed_out_q_scale3[(((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)) + 192)]); } else { condval_23 = (half)0.000000e+00f; } NT_matmul_intermediate_reindex_pad_local[11] = (NT_matmul_intermediate_reindex_pad_local[11] + (condval_22 * condval_23)); half condval_24; if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { condval_24 = take1[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; } else { condval_24 = (half)0.000000e+00f; } half condval_25; if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 4) < vocab_size)) { condval_25 = (((convert_half(((embed_out_q_weight3[((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)) + 1024)] >> cse_var_1) & (uint)15))) - (half)7.000000e+00f) * embed_out_q_scale3[(((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)) + 256)]); } else { condval_25 = (half)0.000000e+00f; } NT_matmul_intermediate_reindex_pad_local[12] = (NT_matmul_intermediate_reindex_pad_local[12] + (condval_24 * condval_25)); half condval_26; if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { condval_26 = take1[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; } else { condval_26 = (half)0.000000e+00f; } half condval_27; if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 5) < vocab_size)) { condval_27 = (((convert_half(((embed_out_q_weight3[((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)) + 1280)] >> cse_var_1) & (uint)15))) - (half)7.000000e+00f) * embed_out_q_scale3[(((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)) + 320)]); } else { condval_27 = (half)0.000000e+00f; } NT_matmul_intermediate_reindex_pad_local[13] = (NT_matmul_intermediate_reindex_pad_local[13] + (condval_26 * condval_27)); half condval_28; if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { condval_28 = take1[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; } else { condval_28 = (half)0.000000e+00f; } half condval_29; if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 6) < vocab_size)) { condval_29 = (((convert_half(((embed_out_q_weight3[((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)) + 1536)] >> cse_var_1) & (uint)15))) - (half)7.000000e+00f) * embed_out_q_scale3[(((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)) + 384)]); } else { condval_29 = (half)0.000000e+00f; } NT_matmul_intermediate_reindex_pad_local[14] = (NT_matmul_intermediate_reindex_pad_local[14] + (condval_28 * condval_29)); half condval_30; if ((((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) { condval_30 = take1[((((((convert_int(get_group_id(1))) * 16384) + ((convert_int(get_local_id(1))) * 4096)) + (ax3_0 * 16)) + ax3_1) + 2048)]; } else { condval_30 = (half)0.000000e+00f; } half condval_31; if ((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 7) < vocab_size)) { condval_31 = (((convert_half(((embed_out_q_weight3[((((((convert_int(get_group_id(0))) * 65536) + ((convert_int(get_local_id(0))) * 2048)) + (ax3_0 * 2)) + (ax3_1 >> 3)) + 1792)] >> cse_var_1) & (uint)15))) - (half)7.000000e+00f) * embed_out_q_scale3[(((((convert_int(get_group_id(0))) * 16384) + ((convert_int(get_local_id(0))) * 512)) + (ax3_0 >> 1)) + 448)]); } else { condval_31 = (half)0.000000e+00f; } NT_matmul_intermediate_reindex_pad_local[15] = (NT_matmul_intermediate_reindex_pad_local[15] + (condval_30 * condval_31)); } } if (((bool)((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) && ((bool)((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) < vocab_size))) { compute[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) * vocab_size))] = (convert_float(NT_matmul_intermediate_reindex_pad_local[0])); } if (((bool)((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 1) < vocab_size))) { compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) * vocab_size)) + 1)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[1])); } if (((bool)((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 2) < vocab_size))) { compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) * vocab_size)) + 2)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[2])); } if (((bool)((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 3) < vocab_size))) { compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) * vocab_size)) + 3)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[3])); } if (((bool)((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 4) < vocab_size))) { compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) * vocab_size)) + 4)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[4])); } if (((bool)((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 5) < vocab_size))) { compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) * vocab_size)) + 5)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[5])); } if (((bool)((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 6) < vocab_size))) { compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) * vocab_size)) + 6)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[6])); } if (((bool)((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 7) < vocab_size))) { compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + ((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) * vocab_size)) + 7)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[7])); } if (((bool)(((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) && ((bool)((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) < vocab_size))) { compute[((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) * vocab_size))] = (convert_float(NT_matmul_intermediate_reindex_pad_local[8])); } if (((bool)(((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 1) < vocab_size))) { compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) * vocab_size)) + 1)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[9])); } if (((bool)(((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 2) < vocab_size))) { compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) * vocab_size)) + 2)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[10])); } if (((bool)(((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 3) < vocab_size))) { compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) * vocab_size)) + 3)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[11])); } if (((bool)(((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 4) < vocab_size))) { compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) * vocab_size)) + 4)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[12])); } if (((bool)(((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 5) < vocab_size))) { compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) * vocab_size)) + 5)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[13])); } if (((bool)(((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 6) < vocab_size))) { compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) * vocab_size)) + 6)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[14])); } if (((bool)(((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) < batch_size)) && ((bool)(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + 7) < vocab_size))) { compute[(((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 8)) + (((((convert_int(get_group_id(1))) * 8) + ((convert_int(get_local_id(1))) * 2)) + 1) * vocab_size)) + 7)] = (convert_float(NT_matmul_intermediate_reindex_pad_local[15])); } } } // Function: gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size) { float in_thread_temp_max_shared[1]; __local float red_buf0[256]; __local float temp_max_shared[1]; float in_thread_temp_sum_shared[1]; __local float red_buf0_1[256]; __local float temp_sum_shared[1]; in_thread_temp_max_shared[0] = -3.402823e+38f; for (int ax0_0 = 0; ax0_0 < ((num_chunks + 31) >> 5); ++ax0_0) { if (((ax0_0 * 32) + (convert_int(get_local_id(0)))) < num_chunks) { in_thread_temp_max_shared[0] = max((float)in_thread_temp_max_shared[0], (float)chunked_max[(((ax0_0 * 32) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * num_chunks)) + (convert_int(get_local_id(0))))]); } } barrier(CLK_LOCAL_MEM_FENCE); ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] = in_thread_temp_max_shared[0]; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 16) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] = max((float)((volatile __local float*)red_buf0)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))], (float)((volatile __local float*)red_buf0)[((((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0)))) + 16)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 8) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] = max((float)((volatile __local float*)red_buf0)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))], (float)((volatile __local float*)red_buf0)[((((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0)))) + 8)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 4) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] = max((float)((volatile __local float*)red_buf0)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))], (float)((volatile __local float*)red_buf0)[((((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0)))) + 4)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 2) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] = max((float)((volatile __local float*)red_buf0)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))], (float)((volatile __local float*)red_buf0)[((((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0)))) + 2)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 1) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] = max((float)((volatile __local float*)red_buf0)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))], (float)((volatile __local float*)red_buf0)[((((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0)))) + 1)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) == 0) { temp_max_shared[0] = ((volatile __local float*)red_buf0)[((convert_int(get_local_id(1))) * 32)]; } in_thread_temp_sum_shared[0] = 0.000000e+00f; barrier(CLK_LOCAL_MEM_FENCE); for (int ax0_0_1 = 0; ax0_0_1 < ((num_chunks + 31) >> 5); ++ax0_0_1) { if (((ax0_0_1 * 32) + (convert_int(get_local_id(0)))) < num_chunks) { float condval; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval = exp(((chunked_sum[(((ax0_0_1 * 32) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * num_chunks)) + (convert_int(get_local_id(0))))] + chunked_max[(((ax0_0_1 * 32) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * num_chunks)) + (convert_int(get_local_id(0))))]) - temp_max_shared[0])); } else { condval = ((convert_float((chunked_max[(((ax0_0_1 * 32) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * num_chunks)) + (convert_int(get_local_id(0))))] == temp_max_shared[0]))) * chunked_sum[(((ax0_0_1 * 32) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * num_chunks)) + (convert_int(get_local_id(0))))]); } in_thread_temp_sum_shared[0] = (in_thread_temp_sum_shared[0] + condval); } } barrier(CLK_LOCAL_MEM_FENCE); ((volatile __local float*)red_buf0_1)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] = in_thread_temp_sum_shared[0]; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 16) { ((volatile __local float*)red_buf0_1)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] = (((volatile __local float*)red_buf0_1)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] + ((volatile __local float*)red_buf0_1)[((((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0)))) + 16)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 8) { ((volatile __local float*)red_buf0_1)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] = (((volatile __local float*)red_buf0_1)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] + ((volatile __local float*)red_buf0_1)[((((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0)))) + 8)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 4) { ((volatile __local float*)red_buf0_1)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] = (((volatile __local float*)red_buf0_1)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] + ((volatile __local float*)red_buf0_1)[((((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0)))) + 4)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 2) { ((volatile __local float*)red_buf0_1)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] = (((volatile __local float*)red_buf0_1)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] + ((volatile __local float*)red_buf0_1)[((((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0)))) + 2)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 1) { ((volatile __local float*)red_buf0_1)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] = (((volatile __local float*)red_buf0_1)[(((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0))))] + ((volatile __local float*)red_buf0_1)[((((convert_int(get_local_id(1))) * 32) + (convert_int(get_local_id(0)))) + 1)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) == 0) { temp_sum_shared[0] = ((volatile __local float*)red_buf0_1)[((convert_int(get_local_id(1))) * 32)]; } barrier(CLK_LOCAL_MEM_FENCE); if ((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + (convert_int(get_local_id(0)))) < vocab_size) { float condval_1; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_1 = exp(((A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0))))] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]) - (log(temp_sum_shared[0]) + temp_max_shared[0]))); } else { condval_1 = ((convert_float((A[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0))))] == temp_max_shared[0]))) / temp_sum_shared[0]); } softmax[((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0))))] = condval_1; } if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + (convert_int(get_local_id(0)))) + 256) < vocab_size) { float condval_2; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_2 = exp(((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 256)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]) - (log(temp_sum_shared[0]) + temp_max_shared[0]))); } else { condval_2 = ((convert_float((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 256)] == temp_max_shared[0]))) / temp_sum_shared[0]); } softmax[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 256)] = condval_2; } if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + (convert_int(get_local_id(0)))) + 512) < vocab_size) { float condval_3; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_3 = exp(((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 512)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]) - (log(temp_sum_shared[0]) + temp_max_shared[0]))); } else { condval_3 = ((convert_float((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 512)] == temp_max_shared[0]))) / temp_sum_shared[0]); } softmax[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 512)] = condval_3; } if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + (convert_int(get_local_id(0)))) + 768) < vocab_size) { float condval_4; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_4 = exp(((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 768)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]) - (log(temp_sum_shared[0]) + temp_max_shared[0]))); } else { condval_4 = ((convert_float((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 768)] == temp_max_shared[0]))) / temp_sum_shared[0]); } softmax[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 768)] = condval_4; } if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + (convert_int(get_local_id(0)))) + 1024) < vocab_size) { float condval_5; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_5 = exp(((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1024)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]) - (log(temp_sum_shared[0]) + temp_max_shared[0]))); } else { condval_5 = ((convert_float((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1024)] == temp_max_shared[0]))) / temp_sum_shared[0]); } softmax[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1024)] = condval_5; } if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + (convert_int(get_local_id(0)))) + 1280) < vocab_size) { float condval_6; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_6 = exp(((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1280)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]) - (log(temp_sum_shared[0]) + temp_max_shared[0]))); } else { condval_6 = ((convert_float((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1280)] == temp_max_shared[0]))) / temp_sum_shared[0]); } softmax[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1280)] = condval_6; } if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + (convert_int(get_local_id(0)))) + 1536) < vocab_size) { float condval_7; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_7 = exp(((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1536)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]) - (log(temp_sum_shared[0]) + temp_max_shared[0]))); } else { condval_7 = ((convert_float((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1536)] == temp_max_shared[0]))) / temp_sum_shared[0]); } softmax[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1536)] = condval_7; } if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + (convert_int(get_local_id(0)))) + 1792) < vocab_size) { float condval_8; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_8 = exp(((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1792)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]) - (log(temp_sum_shared[0]) + temp_max_shared[0]))); } else { condval_8 = ((convert_float((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1792)] == temp_max_shared[0]))) / temp_sum_shared[0]); } softmax[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 1792)] = condval_8; } if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + (convert_int(get_local_id(0)))) + 2048) < vocab_size) { float condval_9; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_9 = exp(((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2048)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]) - (log(temp_sum_shared[0]) + temp_max_shared[0]))); } else { condval_9 = ((convert_float((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2048)] == temp_max_shared[0]))) / temp_sum_shared[0]); } softmax[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2048)] = condval_9; } if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + (convert_int(get_local_id(0)))) + 2304) < vocab_size) { float condval_10; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_10 = exp(((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2304)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]) - (log(temp_sum_shared[0]) + temp_max_shared[0]))); } else { condval_10 = ((convert_float((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2304)] == temp_max_shared[0]))) / temp_sum_shared[0]); } softmax[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2304)] = condval_10; } if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + (convert_int(get_local_id(0)))) + 2560) < vocab_size) { float condval_11; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_11 = exp(((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2560)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]) - (log(temp_sum_shared[0]) + temp_max_shared[0]))); } else { condval_11 = ((convert_float((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2560)] == temp_max_shared[0]))) / temp_sum_shared[0]); } softmax[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2560)] = condval_11; } if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + (convert_int(get_local_id(0)))) + 2816) < vocab_size) { float condval_12; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_12 = exp(((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2816)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]) - (log(temp_sum_shared[0]) + temp_max_shared[0]))); } else { condval_12 = ((convert_float((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2816)] == temp_max_shared[0]))) / temp_sum_shared[0]); } softmax[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 2816)] = condval_12; } if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + (convert_int(get_local_id(0)))) + 3072) < vocab_size) { float condval_13; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_13 = exp(((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3072)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]) - (log(temp_sum_shared[0]) + temp_max_shared[0]))); } else { condval_13 = ((convert_float((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3072)] == temp_max_shared[0]))) / temp_sum_shared[0]); } softmax[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3072)] = condval_13; } if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + (convert_int(get_local_id(0)))) + 3328) < vocab_size) { float condval_14; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_14 = exp(((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3328)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]) - (log(temp_sum_shared[0]) + temp_max_shared[0]))); } else { condval_14 = ((convert_float((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3328)] == temp_max_shared[0]))) / temp_sum_shared[0]); } softmax[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3328)] = condval_14; } if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + (convert_int(get_local_id(0)))) + 3584) < vocab_size) { float condval_15; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_15 = exp(((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3584)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]) - (log(temp_sum_shared[0]) + temp_max_shared[0]))); } else { condval_15 = ((convert_float((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3584)] == temp_max_shared[0]))) / temp_sum_shared[0]); } softmax[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3584)] = condval_15; } if (((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + (convert_int(get_local_id(0)))) + 3840) < vocab_size) { float condval_16; if ((1.000000e-05f < temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)])) { condval_16 = exp(((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3840)] / temperature[(((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks)]) - (log(temp_sum_shared[0]) + temp_max_shared[0]))); } else { condval_16 = ((convert_float((A[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3840)] == temp_max_shared[0]))) / temp_sum_shared[0]); } softmax[(((((((convert_int(get_group_id(0))) % num_chunks) * 4096) + ((convert_int(get_local_id(1))) * 32)) + ((((convert_int(get_group_id(0))) % (num_chunks * batch_size)) / num_chunks) * vocab_size)) + (convert_int(get_local_id(0)))) + 3840)] = condval_16; } } // Function: gpt_neox_q4f16_1_scatter_probs_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n) { if ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) < (batch_size * n)) { dst[((indices[(((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % (n * batch_size)) / n)] * n) + ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % n))] = src[(((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0))))]; } } // Function: gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode) { int kv_chunk_len[1]; float st_m[1]; float st_d[1]; float4 O_local[1]; half4 Q_local[1]; float m_prev[1]; __local float4 O_allreduce[256]; __local float md_allreduce[16]; __local half4 K_smem[256]; __local half4 V_smem[256]; float QK_local[4]; float S_reduce_local[1]; __local float red_buf0[256]; float S_local[1]; half4 V_local[1]; float d_prev[1]; float other_m[1]; float other_d[1]; float4 other_o[1]; float exp_mprev[1]; int cur_page_indptr_begin = page_table_indptr[((convert_int(get_group_id(0))) + page_indptr_elem_offset)]; int cur_page_indptr_end = page_table_indptr[(((convert_int(get_group_id(0))) + page_indptr_elem_offset) + 1)]; int condval; if ((cur_page_indptr_begin != cur_page_indptr_end)) { condval = ((((((cur_page_indptr_end * 16) + length_info[((convert_int(get_group_id(0))) + length_info_elem_offset)]) + length_info[(((B * 2) + (convert_int(get_group_id(0)))) + length_info_elem_offset)]) - length_info[(((convert_int(get_group_id(0))) + B) + length_info_elem_offset)]) - (cur_page_indptr_begin * 16)) - 16); } else { condval = 0; } kv_chunk_len[0] = condval; st_m[0] = -5.000000e+04f; st_d[0] = 1.000000e+00f; O_local[0] = ((float4)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)); half4 condval_1; if ((rotary_mode == 1)) { float4 freq = (((float4)(((convert_float(q_rope_position[((convert_int(get_group_id(0))) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[((convert_int(get_group_id(0))) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[((convert_int(get_group_id(0))) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[((convert_int(get_group_id(0))) + q_rope_position_elem_offset)])) * rope_scale))) / pow(((float4)(rope_theta, rope_theta, rope_theta, rope_theta)), ((convert_float4(((int4)(((((convert_int(get_local_id(0))) & 15) * 8))+(2*0), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*1), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*2), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*3))))) * ((float4)(7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f))))); half4 condval_2; if (((convert_int(get_local_id(0))) < 16)) { condval_2 = (vload4(0, Q + (((((convert_int(get_group_id(0))) * 2048) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + 64)) * ((half4)((half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f))); } else { condval_2 = vload4(0, Q + (((((convert_int(get_group_id(0))) * 2048) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) - 64)); } condval_1 = (convert_half4(((cos(freq) * (convert_float4(vload4(0, Q + ((((convert_int(get_group_id(0))) * 2048) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)))))) + (sin(freq) * (convert_float4(condval_2)))))); } else { condval_1 = vload4(0, Q + ((((convert_int(get_group_id(0))) * 2048) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); } Q_local[0] = condval_1; for (int iterator = 0; iterator < ((kv_chunk_len[0] + 7) >> 3); ++iterator) { if (((iterator * 8) + (convert_int(get_local_id(2)))) < kv_chunk_len[0]) { int condval_3; if ((((iterator * 8) + (convert_int(get_local_id(2)))) < length_info[(((B * 2) + (convert_int(get_group_id(0)))) + length_info_elem_offset)])) { condval_3 = ((iterator * 8) + (convert_int(get_local_id(2)))); } else { condval_3 = ((((iterator * 8) + (convert_int(get_local_id(2)))) + length_info[(((convert_int(get_group_id(0))) + B) + length_info_elem_offset)]) - length_info[(((B * 2) + (convert_int(get_group_id(0)))) + length_info_elem_offset)]); } int seq_offset = condval_3; int page_no = page_table_values[(((seq_offset >> 4) + cur_page_indptr_begin) + page_values_elem_offset)]; half4 condval_4; if ((rotary_mode == 1)) { float4 freq_1 = (((float4)(((convert_float((((iterator * 8) + k_rope_pos_offset[((convert_int(get_group_id(0))) + k_rope_pos_offset_elem_offset)]) + (convert_int(get_local_id(2)))))) * rope_scale), ((convert_float((((iterator * 8) + k_rope_pos_offset[((convert_int(get_group_id(0))) + k_rope_pos_offset_elem_offset)]) + (convert_int(get_local_id(2)))))) * rope_scale), ((convert_float((((iterator * 8) + k_rope_pos_offset[((convert_int(get_group_id(0))) + k_rope_pos_offset_elem_offset)]) + (convert_int(get_local_id(2)))))) * rope_scale), ((convert_float((((iterator * 8) + k_rope_pos_offset[((convert_int(get_group_id(0))) + k_rope_pos_offset_elem_offset)]) + (convert_int(get_local_id(2)))))) * rope_scale))) / pow(((float4)(rope_theta, rope_theta, rope_theta, rope_theta)), ((convert_float4(((int4)(((((convert_int(get_local_id(0))) & 15) * 8))+(2*0), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*1), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*2), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*3))))) * ((float4)(7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f))))); int4 v_ = (int4)(((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*0), ((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*1), ((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*2), ((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*3)); half4 condval_5; if (((convert_int(get_local_id(0))) < 16)) { int4 v__1 = (int4)((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 64))+(1*0), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 64))+(1*1), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 64))+(1*2), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 64))+(1*3)); condval_5 = (((half4)(pages[v__1.s0],pages[v__1.s1],pages[v__1.s2],pages[v__1.s3])) * ((half4)((half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f))); } else { int4 v__2 = (int4)((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) - 64))+(1*0), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) - 64))+(1*1), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) - 64))+(1*2), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) - 64))+(1*3)); condval_5 = ((half4)(pages[v__2.s0],pages[v__2.s1],pages[v__2.s2],pages[v__2.s3])); } condval_4 = (convert_half4(((cos(freq_1) * (convert_float4(((half4)(pages[v_.s0],pages[v_.s1],pages[v_.s2],pages[v_.s3]))))) + (sin(freq_1) * (convert_float4(condval_5)))))); } else { int4 v__3 = (int4)(((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*0), ((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*1), ((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*2), ((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*3)); condval_4 = ((half4)(pages[v__3.s0],pages[v__3.s1],pages[v__3.s2],pages[v__3.s3])); } K_smem[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = condval_4; int4 v__4 = (int4)((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 32768))+(1*0), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 32768))+(1*1), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 32768))+(1*2), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 32768))+(1*3)); V_smem[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = ((half4)(pages[v__4.s0],pages[v__4.s1],pages[v__4.s2],pages[v__4.s3])); } else { K_smem[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = ((half4)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)); V_smem[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = ((half4)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)); } barrier(CLK_LOCAL_MEM_FENCE); m_prev[0] = st_m[0]; vstore4(((((convert_float4(Q_local[0])) * (convert_float4(K_smem[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))]))) * ((float4)(attn_score_scaling_factor, attn_score_scaling_factor, attn_score_scaling_factor, attn_score_scaling_factor))) * ((float4)(1.275174e-01f, 1.275174e-01f, 1.275174e-01f, 1.275174e-01f))), 0, QK_local + 0); S_reduce_local[0] = 0.000000e+00f; S_reduce_local[0] = (S_reduce_local[0] + QK_local[0]); S_reduce_local[0] = (S_reduce_local[0] + QK_local[1]); S_reduce_local[0] = (S_reduce_local[0] + QK_local[2]); S_reduce_local[0] = (S_reduce_local[0] + QK_local[3]); barrier(CLK_LOCAL_MEM_FENCE); ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = S_reduce_local[0]; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 16) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0)))) + 16)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 8) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0)))) + 8)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 4) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0)))) + 4)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 2) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0)))) + 2)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(0))) < 1) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0)))) + 1)]); } barrier(CLK_LOCAL_MEM_FENCE); S_local[0] = -5.000000e+04f; if (((iterator * 8) + (convert_int(get_local_id(2)))) < kv_chunk_len[0]) { S_local[0] = ((volatile __local float*)red_buf0)[((convert_int(get_local_id(2))) * 32)]; } st_m[0] = max((float)st_m[0], (float)S_local[0]); float o_scale = exp2((m_prev[0] - st_m[0])); st_d[0] = (st_d[0] * o_scale); S_local[0] = exp2((S_local[0] - st_m[0])); st_d[0] = (st_d[0] + S_local[0]); O_local[0] = (O_local[0] * ((float4)(o_scale, o_scale, o_scale, o_scale))); V_local[0] = V_smem[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))]; O_local[0] = (O_local[0] + ((convert_float4(V_local[0])) * ((float4)(S_local[0], S_local[0], S_local[0], S_local[0])))); } O_allreduce[(((convert_int(get_local_id(2))) * 32) + (convert_int(get_local_id(0))))] = O_local[0]; md_allreduce[((convert_int(get_local_id(2))) * 2)] = st_m[0]; md_allreduce[(((convert_int(get_local_id(2))) * 2) + 1)] = st_d[0]; barrier(CLK_LOCAL_MEM_FENCE); st_m[0] = -5.000000e+04f; st_d[0] = 1.000000e+00f; O_local[0] = ((float4)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f)); for (int j = 0; j < 8; ++j) { m_prev[0] = st_m[0]; d_prev[0] = st_d[0]; other_m[0] = md_allreduce[(j * 2)]; other_d[0] = md_allreduce[((j * 2) + 1)]; other_o[0] = O_allreduce[((j * 32) + (convert_int(get_local_id(0))))]; st_m[0] = max((float)st_m[0], (float)other_m[0]); st_d[0] = ((d_prev[0] * exp2((m_prev[0] - st_m[0]))) + (other_d[0] * exp2((other_m[0] - st_m[0])))); exp_mprev[0] = exp2((m_prev[0] - st_m[0])); other_m[0] = exp2((other_m[0] - st_m[0])); O_local[0] = ((O_local[0] * ((float4)(exp_mprev[0], exp_mprev[0], exp_mprev[0], exp_mprev[0]))) + (other_o[0] * ((float4)(other_m[0], other_m[0], other_m[0], other_m[0])))); } O_local[0] = (O_local[0] / ((float4)(st_d[0], st_d[0], st_d[0], st_d[0]))); vstore4((convert_half4(O_local[0])), 0, output + ((((convert_int(get_group_id(0))) * 2048) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); lse[(((convert_int(get_group_id(0))) * 16) + (convert_int(get_group_id(1))))] = (st_m[0] + log2(st_d[0])); } // Function: gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1 #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len) { half NT_matmul_intermediate_pad_rf_local[64]; half dequantize_local[16]; half NT_matmul_intermediate_pad_rf_local_1[8]; half NT_matmul_intermediate_pad_local[8]; __local half red_buf0[64]; for (int ax0_1_init = 0; ax0_1_init < 4; ++ax0_1_init) { for (int ax1_fused_2_init = 0; ax1_fused_2_init < 2; ++ax1_fused_2_init) { int8 v_ = (int8)((((ax0_1_init * 2) + ax1_fused_2_init))+(8*0), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*1), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*2), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*3), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*4), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*5), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*6), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*7)); half8 v__1 = (half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f); NT_matmul_intermediate_pad_rf_local[v_.s0] = v__1.s0; NT_matmul_intermediate_pad_rf_local[v_.s1] = v__1.s1; NT_matmul_intermediate_pad_rf_local[v_.s2] = v__1.s2; NT_matmul_intermediate_pad_rf_local[v_.s3] = v__1.s3; NT_matmul_intermediate_pad_rf_local[v_.s4] = v__1.s4; NT_matmul_intermediate_pad_rf_local[v_.s5] = v__1.s5; NT_matmul_intermediate_pad_rf_local[v_.s6] = v__1.s6; NT_matmul_intermediate_pad_rf_local[v_.s7] = v__1.s7; } } for (int ax2_fused_0 = 0; ax2_fused_0 < 8; ++ax2_fused_0) { for (int ax0_0 = 0; ax0_0 < 2; ++ax0_0) { dequantize_local[(ax0_0 * 8)] = (((convert_half(((gpt_neox_layers_0_attention_query_key_value_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)0) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_query_key_value_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 1)] = (((convert_half(((gpt_neox_layers_0_attention_query_key_value_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)4) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_query_key_value_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 2)] = (((convert_half(((gpt_neox_layers_0_attention_query_key_value_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)8) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_query_key_value_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 3)] = (((convert_half(((gpt_neox_layers_0_attention_query_key_value_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)12) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_query_key_value_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 4)] = (((convert_half(((gpt_neox_layers_0_attention_query_key_value_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)16) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_query_key_value_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 5)] = (((convert_half(((gpt_neox_layers_0_attention_query_key_value_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)20) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_query_key_value_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 6)] = (((convert_half(((gpt_neox_layers_0_attention_query_key_value_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)24) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_query_key_value_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 7)] = (((convert_half(((gpt_neox_layers_0_attention_query_key_value_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)28) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_query_key_value_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); } if (((convert_int(get_group_id(1))) * 4) < seq_len) { int8 v__2 = (int8)((0)+(8*0), (0)+(8*1), (0)+(8*2), (0)+(8*3), (0)+(8*4), (0)+(8*5), (0)+(8*6), (0)+(8*7)); half8 v__3 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__2.s0],NT_matmul_intermediate_pad_rf_local[v__2.s1],NT_matmul_intermediate_pad_rf_local[v__2.s2],NT_matmul_intermediate_pad_rf_local[v__2.s3],NT_matmul_intermediate_pad_rf_local[v__2.s4],NT_matmul_intermediate_pad_rf_local[v__2.s5],NT_matmul_intermediate_pad_rf_local[v__2.s6],NT_matmul_intermediate_pad_rf_local[v__2.s7])) + (vload8(0, layer_norm98 + ((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, dequantize_local + 0)); NT_matmul_intermediate_pad_rf_local[v__2.s0] = v__3.s0; NT_matmul_intermediate_pad_rf_local[v__2.s1] = v__3.s1; NT_matmul_intermediate_pad_rf_local[v__2.s2] = v__3.s2; NT_matmul_intermediate_pad_rf_local[v__2.s3] = v__3.s3; NT_matmul_intermediate_pad_rf_local[v__2.s4] = v__3.s4; NT_matmul_intermediate_pad_rf_local[v__2.s5] = v__3.s5; NT_matmul_intermediate_pad_rf_local[v__2.s6] = v__3.s6; NT_matmul_intermediate_pad_rf_local[v__2.s7] = v__3.s7; int8 v__4 = (int8)((1)+(8*0), (1)+(8*1), (1)+(8*2), (1)+(8*3), (1)+(8*4), (1)+(8*5), (1)+(8*6), (1)+(8*7)); half8 v__5 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__4.s0],NT_matmul_intermediate_pad_rf_local[v__4.s1],NT_matmul_intermediate_pad_rf_local[v__4.s2],NT_matmul_intermediate_pad_rf_local[v__4.s3],NT_matmul_intermediate_pad_rf_local[v__4.s4],NT_matmul_intermediate_pad_rf_local[v__4.s5],NT_matmul_intermediate_pad_rf_local[v__4.s6],NT_matmul_intermediate_pad_rf_local[v__4.s7])) + (vload8(0, layer_norm98 + ((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, dequantize_local + 8)); NT_matmul_intermediate_pad_rf_local[v__4.s0] = v__5.s0; NT_matmul_intermediate_pad_rf_local[v__4.s1] = v__5.s1; NT_matmul_intermediate_pad_rf_local[v__4.s2] = v__5.s2; NT_matmul_intermediate_pad_rf_local[v__4.s3] = v__5.s3; NT_matmul_intermediate_pad_rf_local[v__4.s4] = v__5.s4; NT_matmul_intermediate_pad_rf_local[v__4.s5] = v__5.s5; NT_matmul_intermediate_pad_rf_local[v__4.s6] = v__5.s6; NT_matmul_intermediate_pad_rf_local[v__4.s7] = v__5.s7; } if ((((convert_int(get_group_id(1))) * 4) + 1) < seq_len) { int8 v__6 = (int8)((2)+(8*0), (2)+(8*1), (2)+(8*2), (2)+(8*3), (2)+(8*4), (2)+(8*5), (2)+(8*6), (2)+(8*7)); half8 v__7 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__6.s0],NT_matmul_intermediate_pad_rf_local[v__6.s1],NT_matmul_intermediate_pad_rf_local[v__6.s2],NT_matmul_intermediate_pad_rf_local[v__6.s3],NT_matmul_intermediate_pad_rf_local[v__6.s4],NT_matmul_intermediate_pad_rf_local[v__6.s5],NT_matmul_intermediate_pad_rf_local[v__6.s6],NT_matmul_intermediate_pad_rf_local[v__6.s7])) + (vload8(0, layer_norm98 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)) * vload8(0, dequantize_local + 0)); NT_matmul_intermediate_pad_rf_local[v__6.s0] = v__7.s0; NT_matmul_intermediate_pad_rf_local[v__6.s1] = v__7.s1; NT_matmul_intermediate_pad_rf_local[v__6.s2] = v__7.s2; NT_matmul_intermediate_pad_rf_local[v__6.s3] = v__7.s3; NT_matmul_intermediate_pad_rf_local[v__6.s4] = v__7.s4; NT_matmul_intermediate_pad_rf_local[v__6.s5] = v__7.s5; NT_matmul_intermediate_pad_rf_local[v__6.s6] = v__7.s6; NT_matmul_intermediate_pad_rf_local[v__6.s7] = v__7.s7; int8 v__8 = (int8)((3)+(8*0), (3)+(8*1), (3)+(8*2), (3)+(8*3), (3)+(8*4), (3)+(8*5), (3)+(8*6), (3)+(8*7)); half8 v__9 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__8.s0],NT_matmul_intermediate_pad_rf_local[v__8.s1],NT_matmul_intermediate_pad_rf_local[v__8.s2],NT_matmul_intermediate_pad_rf_local[v__8.s3],NT_matmul_intermediate_pad_rf_local[v__8.s4],NT_matmul_intermediate_pad_rf_local[v__8.s5],NT_matmul_intermediate_pad_rf_local[v__8.s6],NT_matmul_intermediate_pad_rf_local[v__8.s7])) + (vload8(0, layer_norm98 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)) * vload8(0, dequantize_local + 8)); NT_matmul_intermediate_pad_rf_local[v__8.s0] = v__9.s0; NT_matmul_intermediate_pad_rf_local[v__8.s1] = v__9.s1; NT_matmul_intermediate_pad_rf_local[v__8.s2] = v__9.s2; NT_matmul_intermediate_pad_rf_local[v__8.s3] = v__9.s3; NT_matmul_intermediate_pad_rf_local[v__8.s4] = v__9.s4; NT_matmul_intermediate_pad_rf_local[v__8.s5] = v__9.s5; NT_matmul_intermediate_pad_rf_local[v__8.s6] = v__9.s6; NT_matmul_intermediate_pad_rf_local[v__8.s7] = v__9.s7; } if ((((convert_int(get_group_id(1))) * 4) + 2) < seq_len) { int8 v__10 = (int8)((4)+(8*0), (4)+(8*1), (4)+(8*2), (4)+(8*3), (4)+(8*4), (4)+(8*5), (4)+(8*6), (4)+(8*7)); half8 v__11 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__10.s0],NT_matmul_intermediate_pad_rf_local[v__10.s1],NT_matmul_intermediate_pad_rf_local[v__10.s2],NT_matmul_intermediate_pad_rf_local[v__10.s3],NT_matmul_intermediate_pad_rf_local[v__10.s4],NT_matmul_intermediate_pad_rf_local[v__10.s5],NT_matmul_intermediate_pad_rf_local[v__10.s6],NT_matmul_intermediate_pad_rf_local[v__10.s7])) + (vload8(0, layer_norm98 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 4096)) * vload8(0, dequantize_local + 0)); NT_matmul_intermediate_pad_rf_local[v__10.s0] = v__11.s0; NT_matmul_intermediate_pad_rf_local[v__10.s1] = v__11.s1; NT_matmul_intermediate_pad_rf_local[v__10.s2] = v__11.s2; NT_matmul_intermediate_pad_rf_local[v__10.s3] = v__11.s3; NT_matmul_intermediate_pad_rf_local[v__10.s4] = v__11.s4; NT_matmul_intermediate_pad_rf_local[v__10.s5] = v__11.s5; NT_matmul_intermediate_pad_rf_local[v__10.s6] = v__11.s6; NT_matmul_intermediate_pad_rf_local[v__10.s7] = v__11.s7; int8 v__12 = (int8)((5)+(8*0), (5)+(8*1), (5)+(8*2), (5)+(8*3), (5)+(8*4), (5)+(8*5), (5)+(8*6), (5)+(8*7)); half8 v__13 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__12.s0],NT_matmul_intermediate_pad_rf_local[v__12.s1],NT_matmul_intermediate_pad_rf_local[v__12.s2],NT_matmul_intermediate_pad_rf_local[v__12.s3],NT_matmul_intermediate_pad_rf_local[v__12.s4],NT_matmul_intermediate_pad_rf_local[v__12.s5],NT_matmul_intermediate_pad_rf_local[v__12.s6],NT_matmul_intermediate_pad_rf_local[v__12.s7])) + (vload8(0, layer_norm98 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 4096)) * vload8(0, dequantize_local + 8)); NT_matmul_intermediate_pad_rf_local[v__12.s0] = v__13.s0; NT_matmul_intermediate_pad_rf_local[v__12.s1] = v__13.s1; NT_matmul_intermediate_pad_rf_local[v__12.s2] = v__13.s2; NT_matmul_intermediate_pad_rf_local[v__12.s3] = v__13.s3; NT_matmul_intermediate_pad_rf_local[v__12.s4] = v__13.s4; NT_matmul_intermediate_pad_rf_local[v__12.s5] = v__13.s5; NT_matmul_intermediate_pad_rf_local[v__12.s6] = v__13.s6; NT_matmul_intermediate_pad_rf_local[v__12.s7] = v__13.s7; } if ((((convert_int(get_group_id(1))) * 4) + 3) < seq_len) { int8 v__14 = (int8)((6)+(8*0), (6)+(8*1), (6)+(8*2), (6)+(8*3), (6)+(8*4), (6)+(8*5), (6)+(8*6), (6)+(8*7)); half8 v__15 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__14.s0],NT_matmul_intermediate_pad_rf_local[v__14.s1],NT_matmul_intermediate_pad_rf_local[v__14.s2],NT_matmul_intermediate_pad_rf_local[v__14.s3],NT_matmul_intermediate_pad_rf_local[v__14.s4],NT_matmul_intermediate_pad_rf_local[v__14.s5],NT_matmul_intermediate_pad_rf_local[v__14.s6],NT_matmul_intermediate_pad_rf_local[v__14.s7])) + (vload8(0, layer_norm98 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 6144)) * vload8(0, dequantize_local + 0)); NT_matmul_intermediate_pad_rf_local[v__14.s0] = v__15.s0; NT_matmul_intermediate_pad_rf_local[v__14.s1] = v__15.s1; NT_matmul_intermediate_pad_rf_local[v__14.s2] = v__15.s2; NT_matmul_intermediate_pad_rf_local[v__14.s3] = v__15.s3; NT_matmul_intermediate_pad_rf_local[v__14.s4] = v__15.s4; NT_matmul_intermediate_pad_rf_local[v__14.s5] = v__15.s5; NT_matmul_intermediate_pad_rf_local[v__14.s6] = v__15.s6; NT_matmul_intermediate_pad_rf_local[v__14.s7] = v__15.s7; int8 v__16 = (int8)((7)+(8*0), (7)+(8*1), (7)+(8*2), (7)+(8*3), (7)+(8*4), (7)+(8*5), (7)+(8*6), (7)+(8*7)); half8 v__17 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__16.s0],NT_matmul_intermediate_pad_rf_local[v__16.s1],NT_matmul_intermediate_pad_rf_local[v__16.s2],NT_matmul_intermediate_pad_rf_local[v__16.s3],NT_matmul_intermediate_pad_rf_local[v__16.s4],NT_matmul_intermediate_pad_rf_local[v__16.s5],NT_matmul_intermediate_pad_rf_local[v__16.s6],NT_matmul_intermediate_pad_rf_local[v__16.s7])) + (vload8(0, layer_norm98 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 6144)) * vload8(0, dequantize_local + 8)); NT_matmul_intermediate_pad_rf_local[v__16.s0] = v__17.s0; NT_matmul_intermediate_pad_rf_local[v__16.s1] = v__17.s1; NT_matmul_intermediate_pad_rf_local[v__16.s2] = v__17.s2; NT_matmul_intermediate_pad_rf_local[v__16.s3] = v__17.s3; NT_matmul_intermediate_pad_rf_local[v__16.s4] = v__17.s4; NT_matmul_intermediate_pad_rf_local[v__16.s5] = v__17.s5; NT_matmul_intermediate_pad_rf_local[v__16.s6] = v__17.s6; NT_matmul_intermediate_pad_rf_local[v__16.s7] = v__17.s7; } } for (int ax3_fused_2_0 = 0; ax3_fused_2_0 < 1; ++ax3_fused_2_0) { for (int ax2 = 0; ax2 < 4; ++ax2) { vstore2(((half2)((half)0.000000e+00f, (half)0.000000e+00f)), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + (ax2 * 2))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 8))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 16))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 24))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 32))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 40))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 48))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 56))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); } } for (int ax2_fused_2 = 0; ax2_fused_2 < 2; ++ax2_fused_2) { for (int ax1 = 0; ax1 < 4; ++ax1) { barrier(CLK_LOCAL_MEM_FENCE); ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_intermediate_pad_rf_local_1[((ax1 * 2) + ax2_fused_2)]; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 16) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 8) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 4) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 2) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); } barrier(CLK_LOCAL_MEM_FENCE); NT_matmul_intermediate_pad_local[((ax1 * 2) + ax2_fused_2)] = ((volatile __local half*)red_buf0)[((convert_int(get_local_id(0))) * 32)]; } } for (int ax0 = 0; ax0 < 4; ++ax0) { for (int ax1_fused_2 = 0; ax1_fused_2 < 2; ++ax1_fused_2) { if (((bool)(((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)(((bool)(((convert_int(get_group_id(1))) - ((seq_len + 3) >> 2)) < 0)) || ((bool)((convert_int(get_group_id(1))) == 0)))))) && ((bool)((((convert_int(get_group_id(1))) * 4) + ax0) < seq_len))) { T_add[((((((convert_int(get_group_id(1))) * 24576) + (ax0 * 6144)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] = (NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_attention_query_key_value_bias3[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]); } } } } // Function: gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len) { int tile_id[1]; int batch_idx[1]; int batch_rows[1]; int batch_tiles[1]; int kv_chunk_len[1]; __local float m_smem[32]; __local float d_smem[32]; float O_local[32]; __local half Q_smem[4096]; __local half K_smem[2048]; __local half V_smem[2048]; float S_local[4]; float m_prev[1]; float m_new[1]; float d_new[1]; __local float m_prev_smem[32]; tile_id[0] = (convert_int(get_group_id(0))); batch_idx[0] = 0; batch_rows[0] = (q_indptr[(q_indptr_elem_offset + 1)] - q_indptr[q_indptr_elem_offset]); batch_tiles[0] = ((batch_rows[0] + 31) >> 5); while (1) { if (!(((batch_idx[0] < batch_size)))) { break; } while (1) { if (!((((bool)(batch_tiles[0] <= tile_id[0])) && ((bool)(batch_idx[0] < batch_size))))) { break; } tile_id[0] = (tile_id[0] - batch_tiles[0]); batch_idx[0] = (batch_idx[0] + 1); if (batch_idx[0] < batch_size) { int b_idx = batch_idx[0]; batch_rows[0] = (q_indptr[((b_idx + q_indptr_elem_offset) + 1)] - q_indptr[(b_idx + q_indptr_elem_offset)]); batch_tiles[0] = ((batch_rows[0] + 31) >> 5); } } if ((batch_idx[0] < batch_size)) { int b_idx_1 = batch_idx[0]; int LH_start = (tile_id[0] * 32); int q_indptr_val = q_indptr[(b_idx_1 + q_indptr_elem_offset)]; int cur_page_indptr_begin = page_indptr[(b_idx_1 + page_indptr_elem_offset)]; int cur_page_indptr_end = page_indptr[((b_idx_1 + page_indptr_elem_offset) + 1)]; int condval; if ((cur_page_indptr_begin != cur_page_indptr_end)) { condval = ((((((cur_page_indptr_end * 16) + length_info[(b_idx_1 + length_info_elem_offset)]) + length_info[(((batch_size * 2) + b_idx_1) + length_info_elem_offset)]) - length_info[((batch_size + b_idx_1) + length_info_elem_offset)]) - (cur_page_indptr_begin * 16)) - 16); } else { condval = 0; } kv_chunk_len[0] = condval; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { m_smem[(convert_int(get_local_id(0)))] = -5.000000e+04f; d_smem[(convert_int(get_local_id(0)))] = 1.000000e+00f; } for (int li_1 = 0; li_1 < 4; ++li_1) { for (int lj_1 = 0; lj_1 < 8; ++lj_1) { O_local[((li_1 * 8) + lj_1)] = 0.000000e+00f; } } barrier(CLK_LOCAL_MEM_FENCE); for (int li_lj_fused_0 = 0; li_lj_fused_0 < 8; ++li_lj_fused_0) { if (((((li_lj_fused_0 * 4) + q_indptr_val) + LH_start) + (convert_int(get_local_id(1)))) < q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)]) { half4 condval_1; if ((rotary_mode == 1)) { float4 freq = (((float4)(((convert_float(q_rope_position[(((((li_lj_fused_0 * 4) + (convert_int(get_local_id(1)))) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[(((((li_lj_fused_0 * 4) + (convert_int(get_local_id(1)))) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[(((((li_lj_fused_0 * 4) + (convert_int(get_local_id(1)))) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale), ((convert_float(q_rope_position[(((((li_lj_fused_0 * 4) + (convert_int(get_local_id(1)))) + q_indptr_val) + LH_start) + q_rope_position_elem_offset)])) * rope_scale))) / pow(((float4)(rope_theta, rope_theta, rope_theta, rope_theta)), ((convert_float4(((int4)(((((convert_int(get_local_id(0))) & 15) * 8))+(2*0), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*1), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*2), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*3))))) * ((float4)(7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f))))); half4 condval_2; if (((convert_int(get_local_id(0))) < 16)) { condval_2 = (vload4(0, q + (((((((li_lj_fused_0 * 8192) + ((convert_int(get_local_id(1))) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) + 64)) * ((half4)((half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f))); } else { condval_2 = vload4(0, q + (((((((li_lj_fused_0 * 8192) + ((convert_int(get_local_id(1))) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)) - 64)); } condval_1 = (convert_half4(((cos(freq) * (convert_float4(vload4(0, q + ((((((li_lj_fused_0 * 8192) + ((convert_int(get_local_id(1))) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4)))))) + (sin(freq) * (convert_float4(condval_2)))))); } else { condval_1 = vload4(0, q + ((((((li_lj_fused_0 * 8192) + ((convert_int(get_local_id(1))) * 2048)) + (q_indptr_val * 2048)) + (LH_start * 2048)) + ((convert_int(get_group_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); } vstore4(condval_1, 0, Q_smem + (((li_lj_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); } else { vstore4(((half4)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, Q_smem + (((li_lj_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); } } barrier(CLK_LOCAL_MEM_FENCE); for (int iterator = 0; iterator < ((kv_chunk_len[0] + 15) >> 4); ++iterator) { barrier(CLK_LOCAL_MEM_FENCE); for (int lz_ly_fused_0 = 0; lz_ly_fused_0 < 4; ++lz_ly_fused_0) { if ((((iterator * 16) + (lz_ly_fused_0 * 4)) + (convert_int(get_local_id(1)))) < kv_chunk_len[0]) { int condval_3; if (((((iterator * 16) + (lz_ly_fused_0 * 4)) + (convert_int(get_local_id(1)))) < length_info[(((batch_size * 2) + b_idx_1) + length_info_elem_offset)])) { condval_3 = (((iterator * 16) + (lz_ly_fused_0 * 4)) + (convert_int(get_local_id(1)))); } else { condval_3 = (((((iterator * 16) + (lz_ly_fused_0 * 4)) + (convert_int(get_local_id(1)))) + length_info[((batch_size + b_idx_1) + length_info_elem_offset)]) - length_info[(((batch_size * 2) + b_idx_1) + length_info_elem_offset)]); } int seq_offset = condval_3; int page_no = page_values[(((seq_offset >> 4) + cur_page_indptr_begin) + page_values_elem_offset)]; half4 condval_4; if ((rotary_mode == 1)) { float4 freq_1 = (((float4)(((convert_float(((((iterator * 16) + (lz_ly_fused_0 * 4)) + k_rope_pos_offset[(b_idx_1 + k_rope_pos_offset_elem_offset)]) + (convert_int(get_local_id(1)))))) * rope_scale), ((convert_float(((((iterator * 16) + (lz_ly_fused_0 * 4)) + k_rope_pos_offset[(b_idx_1 + k_rope_pos_offset_elem_offset)]) + (convert_int(get_local_id(1)))))) * rope_scale), ((convert_float(((((iterator * 16) + (lz_ly_fused_0 * 4)) + k_rope_pos_offset[(b_idx_1 + k_rope_pos_offset_elem_offset)]) + (convert_int(get_local_id(1)))))) * rope_scale), ((convert_float(((((iterator * 16) + (lz_ly_fused_0 * 4)) + k_rope_pos_offset[(b_idx_1 + k_rope_pos_offset_elem_offset)]) + (convert_int(get_local_id(1)))))) * rope_scale))) / pow(((float4)(rope_theta, rope_theta, rope_theta, rope_theta)), ((convert_float4(((int4)(((((convert_int(get_local_id(0))) & 15) * 8))+(2*0), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*1), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*2), ((((convert_int(get_local_id(0))) & 15) * 8))+(2*3))))) * ((float4)(7.812500e-03f, 7.812500e-03f, 7.812500e-03f, 7.812500e-03f))))); int4 v_ = (int4)(((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*0), ((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*1), ((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*2), ((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*3)); half4 condval_5; if (((convert_int(get_local_id(0))) < 16)) { int4 v__1 = (int4)((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 64))+(1*0), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 64))+(1*1), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 64))+(1*2), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 64))+(1*3)); condval_5 = (((half4)(pages[v__1.s0],pages[v__1.s1],pages[v__1.s2],pages[v__1.s3])) * ((half4)((half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f, (half)-1.000000e+00f))); } else { int4 v__2 = (int4)((((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) - 64))+(1*0), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) - 64))+(1*1), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) - 64))+(1*2), (((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) - 64))+(1*3)); condval_5 = ((half4)(pages[v__2.s0],pages[v__2.s1],pages[v__2.s2],pages[v__2.s3])); } condval_4 = (convert_half4(((cos(freq_1) * (convert_float4(((half4)(pages[v_.s0],pages[v_.s1],pages[v_.s2],pages[v_.s3]))))) + (sin(freq_1) * (convert_float4(condval_5)))))); } else { int4 v__3 = (int4)(((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*0), ((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*1), ((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*2), ((((((page_no * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset))+(1*3)); condval_4 = ((half4)(pages[v__3.s0],pages[v__3.s1],pages[v__3.s2],pages[v__3.s3])); } vstore4(condval_4, 0, K_smem + (((lz_ly_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); } else { vstore4(((half4)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, K_smem + (((lz_ly_fused_0 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); } } barrier(CLK_LOCAL_MEM_FENCE); for (int lz_ly_fused_0_1 = 0; lz_ly_fused_0_1 < 4; ++lz_ly_fused_0_1) { if ((((iterator * 16) + (lz_ly_fused_0_1 * 4)) + (convert_int(get_local_id(1)))) < kv_chunk_len[0]) { int condval_6; if (((((iterator * 16) + (lz_ly_fused_0_1 * 4)) + (convert_int(get_local_id(1)))) < length_info[(((batch_size * 2) + b_idx_1) + length_info_elem_offset)])) { condval_6 = (((iterator * 16) + (lz_ly_fused_0_1 * 4)) + (convert_int(get_local_id(1)))); } else { condval_6 = (((((iterator * 16) + (lz_ly_fused_0_1 * 4)) + (convert_int(get_local_id(1)))) + length_info[((batch_size + b_idx_1) + length_info_elem_offset)]) - length_info[(((batch_size * 2) + b_idx_1) + length_info_elem_offset)]); } int seq_offset_1 = condval_6; int page_no_1 = page_values[(((seq_offset_1 >> 4) + cur_page_indptr_begin) + page_values_elem_offset)]; int4 v__4 = (int4)((((((((page_no_1 * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset_1 & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 32768))+(1*0), (((((((page_no_1 * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset_1 & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 32768))+(1*1), (((((((page_no_1 * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset_1 & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 32768))+(1*2), (((((((page_no_1 * 65536) + ((convert_int(get_group_id(1))) * 2048)) + ((seq_offset_1 & 15) * 128)) + ((convert_int(get_local_id(0))) * 4)) + pages_elem_offset) + 32768))+(1*3)); vstore4(((half4)(pages[v__4.s0],pages[v__4.s1],pages[v__4.s2],pages[v__4.s3])), 0, V_smem + (((lz_ly_fused_0_1 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); } else { vstore4(((half4)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f)), 0, V_smem + (((lz_ly_fused_0_1 * 512) + ((convert_int(get_local_id(1))) * 128)) + ((convert_int(get_local_id(0))) * 4))); } } barrier(CLK_LOCAL_MEM_FENCE); for (int li_1_init = 0; li_1_init < 2; ++li_1_init) { for (int lj_1_init = 0; lj_1_init < 2; ++lj_1_init) { S_local[((li_1_init * 2) + lj_1_init)] = 0.000000e+00f; } } for (int lk_0 = 0; lk_0 < 16; ++lk_0) { for (int li_1_1 = 0; li_1_1 < 2; ++li_1_1) { for (int lj_1_1 = 0; lj_1_1 < 2; ++lj_1_1) { for (int lk_1 = 0; lk_1 < 8; ++lk_1) { S_local[((li_1_1 * 2) + lj_1_1)] = (S_local[((li_1_1 * 2) + lj_1_1)] + ((((convert_float(Q_smem[((((((convert_int(get_local_id(1))) * 1024) + (((convert_int(get_local_id(0))) >> 3) * 256)) + (li_1_1 * 128)) + (lk_0 * 8)) + lk_1)])) * (convert_float(K_smem[((((((convert_int(get_local_id(0))) & 7) * 256) + (lj_1_1 * 128)) + (lk_0 * 8)) + lk_1)]))) * attn_score_scaling_factor) * 1.275174e-01f)); } } } } barrier(CLK_LOCAL_MEM_FENCE); for (int li_1_2 = 0; li_1_2 < 2; ++li_1_2) { for (int lj_1_2 = 0; lj_1_2 < 2; ++lj_1_2) { ((__local float*)K_smem)[((((((convert_int(get_local_id(1))) * 128) + (((convert_int(get_local_id(0))) >> 3) * 32)) + (li_1_2 * 16)) + (((convert_int(get_local_id(0))) & 7) * 2)) + lj_1_2)] = S_local[((li_1_2 * 2) + lj_1_2)]; } } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { m_prev[0] = m_smem[(convert_int(get_local_id(0)))]; m_new[0] = m_smem[(convert_int(get_local_id(0)))]; for (int j = 0; j < 16; ++j) { int cse_var_4 = ((iterator * 16) + j); bool condval_7; if ((0 < causal)) { condval_7 = (cse_var_4 <= (((((((convert_int(get_local_id(1))) * 32) + kv_chunk_len[0]) + q_indptr[(b_idx_1 + q_indptr_elem_offset)]) + LH_start) + (convert_int(get_local_id(0)))) - q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)])); } else { condval_7 = (cse_var_4 < kv_chunk_len[0]); } if (condval_7) { m_new[0] = max((float)m_new[0], (float)((__local float*)K_smem)[((((convert_int(get_local_id(1))) * 512) + ((convert_int(get_local_id(0))) * 16)) + j)]); } } d_new[0] = (d_smem[(convert_int(get_local_id(0)))] * exp2((m_prev[0] - m_new[0]))); } for (int j_1 = 0; j_1 < 16; ++j_1) { barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { int cse_var_5 = ((iterator * 16) + j_1); bool condval_8; if ((0 < causal)) { condval_8 = (cse_var_5 <= (((((((convert_int(get_local_id(1))) * 32) + kv_chunk_len[0]) + q_indptr[(b_idx_1 + q_indptr_elem_offset)]) + LH_start) + (convert_int(get_local_id(0)))) - q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)])); } else { condval_8 = (cse_var_5 < kv_chunk_len[0]); } if (condval_8) { ((__local float*)K_smem)[((((convert_int(get_local_id(1))) * 512) + ((convert_int(get_local_id(0))) * 16)) + j_1)] = exp2((((__local float*)K_smem)[((((convert_int(get_local_id(1))) * 512) + ((convert_int(get_local_id(0))) * 16)) + j_1)] - m_new[0])); } else { ((__local float*)K_smem)[(((convert_int(get_local_id(0))) * 16) + j_1)] = exp2((-5.000000e+04f - m_new[0])); } } } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { for (int j_2 = 0; j_2 < 16; ++j_2) { d_new[0] = (d_new[0] + ((__local float*)K_smem)[(((convert_int(get_local_id(0))) * 16) + j_2)]); } m_smem[(convert_int(get_local_id(0)))] = m_new[0]; d_smem[(convert_int(get_local_id(0)))] = d_new[0]; m_prev_smem[(convert_int(get_local_id(0)))] = m_prev[0]; } barrier(CLK_LOCAL_MEM_FENCE); for (int li_1_init_1 = 0; li_1_init_1 < 4; ++li_1_init_1) { for (int lj_1_init_1 = 0; lj_1_init_1 < 8; ++lj_1_init_1) { O_local[((li_1_init_1 * 8) + lj_1_init_1)] = (O_local[((li_1_init_1 * 8) + lj_1_init_1)] * exp2((m_prev_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_init_1)] - m_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_init_1)]))); } } for (int lk_0_1 = 0; lk_0_1 < 2; ++lk_0_1) { for (int lk_1_1 = 0; lk_1_1 < 8; ++lk_1_1) { for (int li_1_3 = 0; li_1_3 < 4; ++li_1_3) { for (int lj_1_3 = 0; lj_1_3 < 8; ++lj_1_3) { O_local[((li_1_3 * 8) + lj_1_3)] = (O_local[((li_1_3 * 8) + lj_1_3)] + (((__local float*)K_smem)[((((((convert_int(get_local_id(1))) * 128) + (((convert_int(get_local_id(0))) >> 4) * 64)) + (li_1_3 * 16)) + (lk_0_1 * 8)) + lk_1_1)] * (convert_float(V_smem[((((lk_0_1 * 1024) + (lk_1_1 * 128)) + (((convert_int(get_local_id(0))) & 15) * 8)) + lj_1_3)])))); } } } } } for (int li_1_4 = 0; li_1_4 < 4; ++li_1_4) { for (int lj_1_4 = 0; lj_1_4 < 8; ++lj_1_4) { int cur_L = ((((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + q_indptr[(b_idx_1 + q_indptr_elem_offset)]) + LH_start) + li_1_4); if (cur_L < q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)]) { output[((((cur_L * 2048) + ((convert_int(get_group_id(1))) * 128)) + (((convert_int(get_local_id(0))) & 15) * 8)) + lj_1_4)] = (convert_half((O_local[((li_1_4 * 8) + lj_1_4)] / d_smem[((((convert_int(get_local_id(1))) * 8) + (((convert_int(get_local_id(0))) >> 4) * 4)) + li_1_4)]))); } } } if ((convert_int(get_local_id(1))) < 1) { int cur_L_1 = (((((convert_int(get_local_id(1))) * 32) + q_indptr[(b_idx_1 + q_indptr_elem_offset)]) + LH_start) + (convert_int(get_local_id(0)))); if (cur_L_1 < q_indptr[((b_idx_1 + q_indptr_elem_offset) + 1)]) { lse[((cur_L_1 * 16) + (convert_int(get_group_id(1))))] = (m_smem[(convert_int(get_local_id(0)))] + log2(d_smem[(convert_int(get_local_id(0)))])); } } tile_id[0] = (tile_id[0] + 16); } } } // Function: gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1 #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size) { half NT_matmul_intermediate_pad_rf_local[64]; half dequantize_local[16]; half NT_matmul_intermediate_pad_rf_local_1[8]; half NT_matmul_intermediate_pad_local[8]; __local half red_buf0[64]; for (int ax0_1_init = 0; ax0_1_init < 4; ++ax0_1_init) { for (int ax1_fused_2_init = 0; ax1_fused_2_init < 2; ++ax1_fused_2_init) { int8 v_ = (int8)((((ax0_1_init * 2) + ax1_fused_2_init))+(8*0), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*1), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*2), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*3), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*4), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*5), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*6), (((ax0_1_init * 2) + ax1_fused_2_init))+(8*7)); half8 v__1 = (half8)((half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f, (half)0.000000e+00f); NT_matmul_intermediate_pad_rf_local[v_.s0] = v__1.s0; NT_matmul_intermediate_pad_rf_local[v_.s1] = v__1.s1; NT_matmul_intermediate_pad_rf_local[v_.s2] = v__1.s2; NT_matmul_intermediate_pad_rf_local[v_.s3] = v__1.s3; NT_matmul_intermediate_pad_rf_local[v_.s4] = v__1.s4; NT_matmul_intermediate_pad_rf_local[v_.s5] = v__1.s5; NT_matmul_intermediate_pad_rf_local[v_.s6] = v__1.s6; NT_matmul_intermediate_pad_rf_local[v_.s7] = v__1.s7; } } for (int ax2_fused_0 = 0; ax2_fused_0 < 8; ++ax2_fused_0) { for (int ax0_0 = 0; ax0_0 < 2; ++ax0_0) { dequantize_local[(ax0_0 * 8)] = (((convert_half(((gpt_neox_layers_0_attention_dense_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)0) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_dense_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 1)] = (((convert_half(((gpt_neox_layers_0_attention_dense_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)4) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_dense_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 2)] = (((convert_half(((gpt_neox_layers_0_attention_dense_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)8) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_dense_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 3)] = (((convert_half(((gpt_neox_layers_0_attention_dense_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)12) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_dense_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 4)] = (((convert_half(((gpt_neox_layers_0_attention_dense_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)16) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_dense_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 5)] = (((convert_half(((gpt_neox_layers_0_attention_dense_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)20) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_dense_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 6)] = (((convert_half(((gpt_neox_layers_0_attention_dense_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)24) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_dense_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 7)] = (((convert_half(((gpt_neox_layers_0_attention_dense_q_weight4[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)28) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_attention_dense_q_scale4[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); } if (((convert_int(get_group_id(1))) * 4) < batch_size) { int8 v__2 = (int8)((0)+(8*0), (0)+(8*1), (0)+(8*2), (0)+(8*3), (0)+(8*4), (0)+(8*5), (0)+(8*6), (0)+(8*7)); half8 v__3 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__2.s0],NT_matmul_intermediate_pad_rf_local[v__2.s1],NT_matmul_intermediate_pad_rf_local[v__2.s2],NT_matmul_intermediate_pad_rf_local[v__2.s3],NT_matmul_intermediate_pad_rf_local[v__2.s4],NT_matmul_intermediate_pad_rf_local[v__2.s5],NT_matmul_intermediate_pad_rf_local[v__2.s6],NT_matmul_intermediate_pad_rf_local[v__2.s7])) + (vload8(0, reshape291 + ((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, dequantize_local + 0)); NT_matmul_intermediate_pad_rf_local[v__2.s0] = v__3.s0; NT_matmul_intermediate_pad_rf_local[v__2.s1] = v__3.s1; NT_matmul_intermediate_pad_rf_local[v__2.s2] = v__3.s2; NT_matmul_intermediate_pad_rf_local[v__2.s3] = v__3.s3; NT_matmul_intermediate_pad_rf_local[v__2.s4] = v__3.s4; NT_matmul_intermediate_pad_rf_local[v__2.s5] = v__3.s5; NT_matmul_intermediate_pad_rf_local[v__2.s6] = v__3.s6; NT_matmul_intermediate_pad_rf_local[v__2.s7] = v__3.s7; int8 v__4 = (int8)((1)+(8*0), (1)+(8*1), (1)+(8*2), (1)+(8*3), (1)+(8*4), (1)+(8*5), (1)+(8*6), (1)+(8*7)); half8 v__5 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__4.s0],NT_matmul_intermediate_pad_rf_local[v__4.s1],NT_matmul_intermediate_pad_rf_local[v__4.s2],NT_matmul_intermediate_pad_rf_local[v__4.s3],NT_matmul_intermediate_pad_rf_local[v__4.s4],NT_matmul_intermediate_pad_rf_local[v__4.s5],NT_matmul_intermediate_pad_rf_local[v__4.s6],NT_matmul_intermediate_pad_rf_local[v__4.s7])) + (vload8(0, reshape291 + ((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))) * vload8(0, dequantize_local + 8)); NT_matmul_intermediate_pad_rf_local[v__4.s0] = v__5.s0; NT_matmul_intermediate_pad_rf_local[v__4.s1] = v__5.s1; NT_matmul_intermediate_pad_rf_local[v__4.s2] = v__5.s2; NT_matmul_intermediate_pad_rf_local[v__4.s3] = v__5.s3; NT_matmul_intermediate_pad_rf_local[v__4.s4] = v__5.s4; NT_matmul_intermediate_pad_rf_local[v__4.s5] = v__5.s5; NT_matmul_intermediate_pad_rf_local[v__4.s6] = v__5.s6; NT_matmul_intermediate_pad_rf_local[v__4.s7] = v__5.s7; } if ((((convert_int(get_group_id(1))) * 4) + 1) < batch_size) { int8 v__6 = (int8)((2)+(8*0), (2)+(8*1), (2)+(8*2), (2)+(8*3), (2)+(8*4), (2)+(8*5), (2)+(8*6), (2)+(8*7)); half8 v__7 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__6.s0],NT_matmul_intermediate_pad_rf_local[v__6.s1],NT_matmul_intermediate_pad_rf_local[v__6.s2],NT_matmul_intermediate_pad_rf_local[v__6.s3],NT_matmul_intermediate_pad_rf_local[v__6.s4],NT_matmul_intermediate_pad_rf_local[v__6.s5],NT_matmul_intermediate_pad_rf_local[v__6.s6],NT_matmul_intermediate_pad_rf_local[v__6.s7])) + (vload8(0, reshape291 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)) * vload8(0, dequantize_local + 0)); NT_matmul_intermediate_pad_rf_local[v__6.s0] = v__7.s0; NT_matmul_intermediate_pad_rf_local[v__6.s1] = v__7.s1; NT_matmul_intermediate_pad_rf_local[v__6.s2] = v__7.s2; NT_matmul_intermediate_pad_rf_local[v__6.s3] = v__7.s3; NT_matmul_intermediate_pad_rf_local[v__6.s4] = v__7.s4; NT_matmul_intermediate_pad_rf_local[v__6.s5] = v__7.s5; NT_matmul_intermediate_pad_rf_local[v__6.s6] = v__7.s6; NT_matmul_intermediate_pad_rf_local[v__6.s7] = v__7.s7; int8 v__8 = (int8)((3)+(8*0), (3)+(8*1), (3)+(8*2), (3)+(8*3), (3)+(8*4), (3)+(8*5), (3)+(8*6), (3)+(8*7)); half8 v__9 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__8.s0],NT_matmul_intermediate_pad_rf_local[v__8.s1],NT_matmul_intermediate_pad_rf_local[v__8.s2],NT_matmul_intermediate_pad_rf_local[v__8.s3],NT_matmul_intermediate_pad_rf_local[v__8.s4],NT_matmul_intermediate_pad_rf_local[v__8.s5],NT_matmul_intermediate_pad_rf_local[v__8.s6],NT_matmul_intermediate_pad_rf_local[v__8.s7])) + (vload8(0, reshape291 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)) * vload8(0, dequantize_local + 8)); NT_matmul_intermediate_pad_rf_local[v__8.s0] = v__9.s0; NT_matmul_intermediate_pad_rf_local[v__8.s1] = v__9.s1; NT_matmul_intermediate_pad_rf_local[v__8.s2] = v__9.s2; NT_matmul_intermediate_pad_rf_local[v__8.s3] = v__9.s3; NT_matmul_intermediate_pad_rf_local[v__8.s4] = v__9.s4; NT_matmul_intermediate_pad_rf_local[v__8.s5] = v__9.s5; NT_matmul_intermediate_pad_rf_local[v__8.s6] = v__9.s6; NT_matmul_intermediate_pad_rf_local[v__8.s7] = v__9.s7; } if ((((convert_int(get_group_id(1))) * 4) + 2) < batch_size) { int8 v__10 = (int8)((4)+(8*0), (4)+(8*1), (4)+(8*2), (4)+(8*3), (4)+(8*4), (4)+(8*5), (4)+(8*6), (4)+(8*7)); half8 v__11 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__10.s0],NT_matmul_intermediate_pad_rf_local[v__10.s1],NT_matmul_intermediate_pad_rf_local[v__10.s2],NT_matmul_intermediate_pad_rf_local[v__10.s3],NT_matmul_intermediate_pad_rf_local[v__10.s4],NT_matmul_intermediate_pad_rf_local[v__10.s5],NT_matmul_intermediate_pad_rf_local[v__10.s6],NT_matmul_intermediate_pad_rf_local[v__10.s7])) + (vload8(0, reshape291 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 4096)) * vload8(0, dequantize_local + 0)); NT_matmul_intermediate_pad_rf_local[v__10.s0] = v__11.s0; NT_matmul_intermediate_pad_rf_local[v__10.s1] = v__11.s1; NT_matmul_intermediate_pad_rf_local[v__10.s2] = v__11.s2; NT_matmul_intermediate_pad_rf_local[v__10.s3] = v__11.s3; NT_matmul_intermediate_pad_rf_local[v__10.s4] = v__11.s4; NT_matmul_intermediate_pad_rf_local[v__10.s5] = v__11.s5; NT_matmul_intermediate_pad_rf_local[v__10.s6] = v__11.s6; NT_matmul_intermediate_pad_rf_local[v__10.s7] = v__11.s7; int8 v__12 = (int8)((5)+(8*0), (5)+(8*1), (5)+(8*2), (5)+(8*3), (5)+(8*4), (5)+(8*5), (5)+(8*6), (5)+(8*7)); half8 v__13 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__12.s0],NT_matmul_intermediate_pad_rf_local[v__12.s1],NT_matmul_intermediate_pad_rf_local[v__12.s2],NT_matmul_intermediate_pad_rf_local[v__12.s3],NT_matmul_intermediate_pad_rf_local[v__12.s4],NT_matmul_intermediate_pad_rf_local[v__12.s5],NT_matmul_intermediate_pad_rf_local[v__12.s6],NT_matmul_intermediate_pad_rf_local[v__12.s7])) + (vload8(0, reshape291 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 4096)) * vload8(0, dequantize_local + 8)); NT_matmul_intermediate_pad_rf_local[v__12.s0] = v__13.s0; NT_matmul_intermediate_pad_rf_local[v__12.s1] = v__13.s1; NT_matmul_intermediate_pad_rf_local[v__12.s2] = v__13.s2; NT_matmul_intermediate_pad_rf_local[v__12.s3] = v__13.s3; NT_matmul_intermediate_pad_rf_local[v__12.s4] = v__13.s4; NT_matmul_intermediate_pad_rf_local[v__12.s5] = v__13.s5; NT_matmul_intermediate_pad_rf_local[v__12.s6] = v__13.s6; NT_matmul_intermediate_pad_rf_local[v__12.s7] = v__13.s7; } if ((((convert_int(get_group_id(1))) * 4) + 3) < batch_size) { int8 v__14 = (int8)((6)+(8*0), (6)+(8*1), (6)+(8*2), (6)+(8*3), (6)+(8*4), (6)+(8*5), (6)+(8*6), (6)+(8*7)); half8 v__15 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__14.s0],NT_matmul_intermediate_pad_rf_local[v__14.s1],NT_matmul_intermediate_pad_rf_local[v__14.s2],NT_matmul_intermediate_pad_rf_local[v__14.s3],NT_matmul_intermediate_pad_rf_local[v__14.s4],NT_matmul_intermediate_pad_rf_local[v__14.s5],NT_matmul_intermediate_pad_rf_local[v__14.s6],NT_matmul_intermediate_pad_rf_local[v__14.s7])) + (vload8(0, reshape291 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 6144)) * vload8(0, dequantize_local + 0)); NT_matmul_intermediate_pad_rf_local[v__14.s0] = v__15.s0; NT_matmul_intermediate_pad_rf_local[v__14.s1] = v__15.s1; NT_matmul_intermediate_pad_rf_local[v__14.s2] = v__15.s2; NT_matmul_intermediate_pad_rf_local[v__14.s3] = v__15.s3; NT_matmul_intermediate_pad_rf_local[v__14.s4] = v__15.s4; NT_matmul_intermediate_pad_rf_local[v__14.s5] = v__15.s5; NT_matmul_intermediate_pad_rf_local[v__14.s6] = v__15.s6; NT_matmul_intermediate_pad_rf_local[v__14.s7] = v__15.s7; int8 v__16 = (int8)((7)+(8*0), (7)+(8*1), (7)+(8*2), (7)+(8*3), (7)+(8*4), (7)+(8*5), (7)+(8*6), (7)+(8*7)); half8 v__17 = ((half8)(NT_matmul_intermediate_pad_rf_local[v__16.s0],NT_matmul_intermediate_pad_rf_local[v__16.s1],NT_matmul_intermediate_pad_rf_local[v__16.s2],NT_matmul_intermediate_pad_rf_local[v__16.s3],NT_matmul_intermediate_pad_rf_local[v__16.s4],NT_matmul_intermediate_pad_rf_local[v__16.s5],NT_matmul_intermediate_pad_rf_local[v__16.s6],NT_matmul_intermediate_pad_rf_local[v__16.s7])) + (vload8(0, reshape291 + (((((convert_int(get_group_id(1))) * 8192) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 6144)) * vload8(0, dequantize_local + 8)); NT_matmul_intermediate_pad_rf_local[v__16.s0] = v__17.s0; NT_matmul_intermediate_pad_rf_local[v__16.s1] = v__17.s1; NT_matmul_intermediate_pad_rf_local[v__16.s2] = v__17.s2; NT_matmul_intermediate_pad_rf_local[v__16.s3] = v__17.s3; NT_matmul_intermediate_pad_rf_local[v__16.s4] = v__17.s4; NT_matmul_intermediate_pad_rf_local[v__16.s5] = v__17.s5; NT_matmul_intermediate_pad_rf_local[v__16.s6] = v__17.s6; NT_matmul_intermediate_pad_rf_local[v__16.s7] = v__17.s7; } } for (int ax3_fused_2_0 = 0; ax3_fused_2_0 < 1; ++ax3_fused_2_0) { for (int ax2 = 0; ax2 < 4; ++ax2) { vstore2(((half2)((half)0.000000e+00f, (half)0.000000e+00f)), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + (ax2 * 2))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 8))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 16))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 24))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 32))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 40))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 48))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 56))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); } } for (int ax2_fused_2 = 0; ax2_fused_2 < 2; ++ax2_fused_2) { for (int ax1 = 0; ax1 < 4; ++ax1) { barrier(CLK_LOCAL_MEM_FENCE); ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_intermediate_pad_rf_local_1[((ax1 * 2) + ax2_fused_2)]; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 16) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 8) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 4) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 2) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { ((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local half*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local half*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); } barrier(CLK_LOCAL_MEM_FENCE); NT_matmul_intermediate_pad_local[((ax1 * 2) + ax2_fused_2)] = ((volatile __local half*)red_buf0)[((convert_int(get_local_id(0))) * 32)]; } } for (int ax0 = 0; ax0 < 4; ++ax0) { for (int ax1_fused_2 = 0; ax1_fused_2 < 2; ++ax1_fused_2) { if (((bool)(((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)(((bool)(((convert_int(get_group_id(1))) - ((batch_size + 3) >> 2)) < 0)) || ((bool)((convert_int(get_group_id(1))) == 0)))))) && ((bool)((((convert_int(get_group_id(1))) * 4) + ax0) < batch_size))) { T_add[((((((convert_int(get_group_id(1))) * 8192) + (ax0 * 2048)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] = ((lv2[((((((convert_int(get_group_id(1))) * 8192) + (ax0 * 2048)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] + (NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_attention_dense_bias4[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)])) + input_embeds[((((((convert_int(get_group_id(1))) * 8192) + (ax0 * 2048)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]); } } } } // Function: gpt_neox_q4f16_1_copy_single_page_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id) { if (((convert_int(get_group_id(0))) - (copy_length * 8)) < 0) { int rmod = ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % (copy_length * 128)); int rmod_1 = ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % (copy_length * 128)); int rdiv = ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) / (copy_length * 128)); int rmod_2 = ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % (copy_length * 128)); int rmod_3 = ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % (copy_length * 128)); int rdiv_1 = ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) / (copy_length * 128)); pages[(((((select((rmod_2 + (copy_length * 128)), rmod_2, (convert_int((((bool)(((bool)((copy_length * 128) >= 0)) && ((bool)(rmod_2 >= 0)))) || ((bool)(((bool)((copy_length * 128) < 0)) && ((bool)(rmod_2 <= 0)))))))) >> 7) * 128) + ((((tgt_page_id * 32) + select((rdiv_1 - 1), rdiv_1, (convert_int((((bool)(((bool)((copy_length * 128) >= 0)) && ((bool)(rmod_3 >= 0)))) || ((bool)(((bool)((copy_length * 128) < 0)) && ((bool)(rmod_3 <= 0))))))))) * page_size) * 128)) + pages_elem_offset) + ((convert_int(get_local_id(0))) & 127))] = pages[(((((select((rmod + (copy_length * 128)), rmod, (convert_int((((bool)(((bool)((copy_length * 128) >= 0)) && ((bool)(rmod >= 0)))) || ((bool)(((bool)((copy_length * 128) < 0)) && ((bool)(rmod <= 0)))))))) >> 7) * 128) + ((((src_page_id * 32) + select((rdiv - 1), rdiv, (convert_int((((bool)(((bool)((copy_length * 128) >= 0)) && ((bool)(rmod_1 >= 0)))) || ((bool)(((bool)((copy_length * 128) < 0)) && ((bool)(rmod_1 <= 0))))))))) * page_size) * 128)) + pages_elem_offset) + ((convert_int(get_local_id(0))) & 127))]; int rmod_4 = ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % (copy_length * 128)); int rmod_5 = ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % (copy_length * 128)); int rdiv_2 = ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) / (copy_length * 128)); int rmod_6 = ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % (copy_length * 128)); int rmod_7 = ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) % (copy_length * 128)); int rdiv_3 = ((((convert_int(get_group_id(0))) * 256) + (convert_int(get_local_id(0)))) / (copy_length * 128)); pages[(((((select((rmod_6 + (copy_length * 128)), rmod_6, (convert_int((((bool)(((bool)((copy_length * 128) >= 0)) && ((bool)(rmod_6 >= 0)))) || ((bool)(((bool)((copy_length * 128) < 0)) && ((bool)(rmod_6 <= 0)))))))) >> 7) * 128) + (((((tgt_page_id * 32) + select((rdiv_3 - 1), rdiv_3, (convert_int((((bool)(((bool)((copy_length * 128) >= 0)) && ((bool)(rmod_7 >= 0)))) || ((bool)(((bool)((copy_length * 128) < 0)) && ((bool)(rmod_7 <= 0))))))))) + 16) * page_size) * 128)) + pages_elem_offset) + ((convert_int(get_local_id(0))) & 127))] = pages[(((((select((rmod_4 + (copy_length * 128)), rmod_4, (convert_int((((bool)(((bool)((copy_length * 128) >= 0)) && ((bool)(rmod_4 >= 0)))) || ((bool)(((bool)((copy_length * 128) < 0)) && ((bool)(rmod_4 <= 0)))))))) >> 7) * 128) + (((((src_page_id * 32) + select((rdiv_2 - 1), rdiv_2, (convert_int((((bool)(((bool)((copy_length * 128) >= 0)) && ((bool)(rmod_5 >= 0)))) || ((bool)(((bool)((copy_length * 128) < 0)) && ((bool)(rmod_5 <= 0))))))))) + 16) * page_size) * 128)) + pages_elem_offset) + ((convert_int(get_local_id(0))) & 127))]; } } // Function: gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel #ifdef cl_khr_fp16 #pragma OPENCL EXTENSION cl_khr_fp16 : enable #elif defined(cl_amd_fp16) #pragma OPENCL EXTENSION cl_amd_fp16 : enable #else #error "Half precision floating point not supported by OpenCL implementation on your device." #endif __kernel void gpt_neox_q4f16_1_tir_kv_cache_transpose_append_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int ntoken, int num_pages, int pages_elem_offset, int position_map_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul12_add12_gelu2_cast6_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight2, __global half* restrict layer_norm50); __kernel void gpt_neox_q4f16_1_reshape4_kernel(__global half* restrict T_reshape, __global half* restrict add288, int seq_len); __kernel void gpt_neox_q4f16_1_reshape2_kernel(__global half* restrict T_reshape, __global half* restrict lv366, int batch_size); __kernel void gpt_neox_q4f16_1_reshape_kernel(__global half* restrict T_reshape, __global half* restrict add432, int batch_size); __kernel void gpt_neox_q4f16_1_tir_kv_cache_debug_get_kv_kernel(__global half* restrict k_data, __global half* restrict pages, __global int* restrict position_map, __global half* restrict v_data, int layer_id, int num_pages, int page_size, int pages_elem_offset, int position_map_elem_offset, int seqlen); __kernel void gpt_neox_q4f16_1_apply_logit_bias_inplace_kernel(__global float* restrict logit_bias, __global float* restrict logits, __global int* restrict pos2seq_id, __global int* restrict token_ids, int batch_size, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_take_kernel(__global half* restrict T_take, __global half* restrict layer_norm146, __global int* restrict logit_positions, int batch_size, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm1_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias3, __global half* restrict gpt_neox_layers_0_input_layernorm_weight3, __global half* restrict input_embeds, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_gather_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_fused_reshape8_reshape9_kernel(__global half* restrict T_reshape, __global half* restrict add144); __kernel void gpt_neox_q4f16_1_fused_rope_kernel(__global half* restrict k, __global int* restrict position_map, __global half* restrict q, __global half* restrict qkv, __global half* restrict v, int apply_rope, int position_map_elem_offset, int seq_len); __kernel void gpt_neox_q4f16_1_reshape3_kernel(__global half* restrict T_reshape, __global half* restrict reshape290, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_reshape10_reshape11_kernel(__global half* restrict T_reshape, __global half* restrict lv124); __kernel void gpt_neox_q4f16_1_layer_norm_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias4, __global half* restrict gpt_neox_layers_0_input_layernorm_weight4, __global half* restrict input_embeds, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul13_add13_cast7_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias2, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale2, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight2, __global half* restrict lv294); __kernel void gpt_neox_q4f16_1_batch_prefill_ragged_kv_kernel(__global half* restrict k, __global int* restrict k_rope_pos_offset, __global int* restrict kv_indptr, __global float* restrict lse, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int kv_indptr_elem_offset, int kv_len, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_chunk_lse_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape7_kernel(__global half* restrict T_reshape, __global half* restrict reshape194, int seq_len); __kernel void gpt_neox_q4f16_1_index_kernel(__global half* restrict index, __global half* restrict layer_norm48, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul6_add6_add9_add9_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias3, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight3, __global half* restrict input_embeds, __global half* restrict lv74, __global half* restrict reshape195, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul10_add10_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias2, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight2, __global half* restrict layer_norm49); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize_take1_kernel(__global half* restrict T_take, __global half* restrict gpt_neox_embed_in_q_scale, __global uint* restrict gpt_neox_embed_in_q_weight, __global int* restrict input_ids, int seq_len, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_tree_attn_paged_kv_kernel(__global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict tree_order, __global int* restrict tree_order_indptr, float attn_score_scaling_factor, int batch_size, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int q_indptr_elem_offset, int total_len, int total_tree_order_len, int tree_order_elem_offset, int tree_order_indptr_elem_offset); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul14_cast8_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale2, __global uint* restrict embed_out_q_weight2, __global half* restrict layer_norm97, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul2_add2_gelu_cast_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight4, __global half* restrict layer_norm148, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul3_add3_cast1_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias4, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale4, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight4, __global half* restrict lv1, int batch_size); __kernel void gpt_neox_q4f16_1_apply_penalty_inplace_kernel(__global float* restrict logits, __global float* restrict penalties, __global int* restrict pos2seq_id, __global int* restrict seq_ids, __global int* restrict token_cnt, __global int* restrict token_ids, int batch_size, int num_seq, int num_token, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape6_kernel(__global half* restrict T_reshape, __global half* restrict lv245, int seq_len); __kernel void gpt_neox_q4f16_1_reshape5_kernel(__global half* restrict T_reshape, __global half* restrict reshape192, int seq_len); __kernel void gpt_neox_q4f16_1_compact_kv_copy_kernel(__global int* restrict copy_length_indptr, __global int* restrict copy_src_dst_pos, __global half* restrict pages, int batch_size, int copy_length_indptr_elem_offset, int copy_src_dst_pos_elem_offset, int num_pages, int pages_elem_offset, int total_copy_length); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_2(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_2(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_apply_bitmask_inplace_kernel(__global int* restrict bitmask, __global float* restrict logits, __global int* restrict seq_ids, int batch_size, int num_seq, int vocab_size); __kernel void gpt_neox_q4f16_1_fused_dequantize4_fused_NT_matmul8_add8_cast4_kernel_1(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_4h_to_h_q_weight3, __global half* restrict lv98, int seq_len); __kernel void gpt_neox_q4f16_1_layer_norm2_kernel(__global half* restrict T_layer_norm, __global half* restrict gpt_neox_layers_0_input_layernorm_bias2, __global half* restrict gpt_neox_layers_0_input_layernorm_weight2, __global half* restrict input_embed); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul_add_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias4, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight4, __global half* restrict layer_norm147, int batch_size); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul11_add11_add14_add14_kernel(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias2, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale2, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight2, __global half* restrict input_embed, __global half* restrict lv218, __global half* restrict lv293); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul4_cast2_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale4, __global uint* restrict embed_out_q_weight4, __global half* restrict layer_norm195, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_reshape1_kernel(__global half* restrict T_reshape, __global half* restrict reshape288, int batch_size); __kernel void gpt_neox_q4f16_1_batch_tree_attn_kernel(__global half* restrict k, __global int* restrict kv_indptr, __global float* restrict lse, __global int* restrict mask, __global int* restrict mn_indptr, __global half* restrict output, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, __global half* restrict v, float attn_score_scaling_factor, int batch_size, int kv_indptr_elem_offset, int kv_len, int mask_elem_offset, int mn_indptr_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, int qo_len, float rope_scale, float rope_theta, int rotary_mode, int tree_size); __kernel void gpt_neox_q4f16_1_merge_state_inplace_kernel(__global float* restrict S, __global float* restrict S_other, __global half* restrict V, __global half* restrict V_other, int D, int H, int N); __kernel void gpt_neox_q4f16_1_fused_dequantize_fused_NT_matmul9_cast5_kernel(__global float* restrict compute, __global half* restrict embed_out_q_scale3, __global uint* restrict embed_out_q_weight3, __global half* restrict take1, int batch_size, int vocab_size); __kernel void gpt_neox_q4f16_1_softmax_with_chunked_sum_kernel(__global float* restrict A, __global float* restrict chunked_max, __global float* restrict chunked_sum, __global float* restrict softmax, __global float* restrict temperature, int batch_size, int num_chunks, int vocab_size); __kernel void gpt_neox_q4f16_1_scatter_probs_kernel(__global float* restrict dst, __global int* restrict indices, __global float* restrict src, int batch_size, int m, int n); __kernel void gpt_neox_q4f16_1_batch_decode_paged_kv_sliding_window_kernel(__global half* restrict Q, __global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_table_indptr, __global int* restrict page_table_values, __global half* restrict pages, __global int* restrict q_rope_position, int B, float attn_score_scaling_factor, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode); __kernel void gpt_neox_q4f16_1_fused_dequantize1_fused_NT_matmul5_add5_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_query_key_value_bias3, __global half* restrict gpt_neox_layers_0_attention_query_key_value_q_scale3, __global uint* restrict gpt_neox_layers_0_attention_query_key_value_q_weight3, __global half* restrict layer_norm98, int seq_len); __kernel void gpt_neox_q4f16_1_batch_prefill_paged_kv_sliding_window_kernel(__global int* restrict k_rope_pos_offset, __global int* restrict length_info, __global float* restrict lse, __global half* restrict output, __global int* restrict page_indptr, __global int* restrict page_values, __global half* restrict pages, __global half* restrict q, __global int* restrict q_indptr, __global int* restrict q_rope_position, float attn_score_scaling_factor, int batch_size, int causal, int k_rope_pos_offset_elem_offset, int length_info_elem_offset, int max_num_pages, int nnz_pages, int page_indptr_elem_offset, int page_values_elem_offset, int pages_elem_offset, int q_indptr_elem_offset, int q_rope_position_elem_offset, float rope_scale, float rope_theta, int rotary_mode, int total_len); __kernel void gpt_neox_q4f16_1_fused_dequantize2_fused_NT_matmul1_add1_add4_add4_kernel_1(__global half* restrict T_add, __global half* restrict gpt_neox_layers_0_attention_dense_bias4, __global half* restrict gpt_neox_layers_0_attention_dense_q_scale4, __global uint* restrict gpt_neox_layers_0_attention_dense_q_weight4, __global half* restrict input_embeds, __global half* restrict lv2, __global half* restrict reshape291, int batch_size); __kernel void gpt_neox_q4f16_1_copy_single_page_kernel(__global half* restrict pages, int copy_length, int num_pages, int page_size, int pages_elem_offset, int src_page_id, int tgt_page_id); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len); __kernel void gpt_neox_q4f16_1_fused_dequantize3_fused_NT_matmul7_add7_gelu1_cast3_kernel(__global half* restrict compute, __global float* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_bias3, __global half* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3, __global uint* restrict gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3, __global half* restrict layer_norm99, int seq_len) { float NT_matmul_intermediate_pad_rf_local[32]; half dequantize_local[16]; float NT_matmul_intermediate_pad_rf_local_1[4]; float NT_matmul_intermediate_pad_local[4]; __local float red_buf0[64]; for (int ax0_1_init = 0; ax0_1_init < 2; ++ax0_1_init) { for (int ax1_fused_2_init = 0; ax1_fused_2_init < 2; ++ax1_fused_2_init) { int8 v_ = (int8)((((ax0_1_init * 2) + ax1_fused_2_init))+(4*0), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*1), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*2), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*3), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*4), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*5), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*6), (((ax0_1_init * 2) + ax1_fused_2_init))+(4*7)); float8 v__1 = (float8)(0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f, 0.000000e+00f); NT_matmul_intermediate_pad_rf_local[v_.s0] = v__1.s0; NT_matmul_intermediate_pad_rf_local[v_.s1] = v__1.s1; NT_matmul_intermediate_pad_rf_local[v_.s2] = v__1.s2; NT_matmul_intermediate_pad_rf_local[v_.s3] = v__1.s3; NT_matmul_intermediate_pad_rf_local[v_.s4] = v__1.s4; NT_matmul_intermediate_pad_rf_local[v_.s5] = v__1.s5; NT_matmul_intermediate_pad_rf_local[v_.s6] = v__1.s6; NT_matmul_intermediate_pad_rf_local[v_.s7] = v__1.s7; } } for (int ax2_fused_0 = 0; ax2_fused_0 < 8; ++ax2_fused_0) { for (int ax0_0 = 0; ax0_0 < 2; ++ax0_0) { dequantize_local[(ax0_0 * 8)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)0) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 1)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)4) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 2)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)8) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 3)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)12) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 4)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)16) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 5)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)20) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 6)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)24) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); dequantize_local[((ax0_0 * 8) + 7)] = (((convert_half(((gpt_neox_layers_0_mlp_dense_h_to_4h_q_weight3[((((((convert_int(get_group_id(0))) * 1024) + ((convert_int(get_local_id(0))) * 512)) + (ax0_0 * 256)) + (ax2_fused_0 * 32)) + (convert_int(get_local_id(1))))] >> (uint)28) & (uint)15))) - (half)7.000000e+00f) * gpt_neox_layers_0_mlp_dense_h_to_4h_q_scale3[((((((convert_int(get_group_id(0))) * 256) + ((convert_int(get_local_id(0))) * 128)) + (ax0_0 * 64)) + (ax2_fused_0 * 8)) + ((convert_int(get_local_id(1))) >> 2))]); } if (((convert_int(get_group_id(1))) * 2) < seq_len) { int8 v__2 = (int8)((0)+(4*0), (0)+(4*1), (0)+(4*2), (0)+(4*3), (0)+(4*4), (0)+(4*5), (0)+(4*6), (0)+(4*7)); float8 v__3 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__2.s0],NT_matmul_intermediate_pad_rf_local[v__2.s1],NT_matmul_intermediate_pad_rf_local[v__2.s2],NT_matmul_intermediate_pad_rf_local[v__2.s3],NT_matmul_intermediate_pad_rf_local[v__2.s4],NT_matmul_intermediate_pad_rf_local[v__2.s5],NT_matmul_intermediate_pad_rf_local[v__2.s6],NT_matmul_intermediate_pad_rf_local[v__2.s7])) + ((convert_float8(vload8(0, layer_norm99 + ((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, dequantize_local + 0)))); NT_matmul_intermediate_pad_rf_local[v__2.s0] = v__3.s0; NT_matmul_intermediate_pad_rf_local[v__2.s1] = v__3.s1; NT_matmul_intermediate_pad_rf_local[v__2.s2] = v__3.s2; NT_matmul_intermediate_pad_rf_local[v__2.s3] = v__3.s3; NT_matmul_intermediate_pad_rf_local[v__2.s4] = v__3.s4; NT_matmul_intermediate_pad_rf_local[v__2.s5] = v__3.s5; NT_matmul_intermediate_pad_rf_local[v__2.s6] = v__3.s6; NT_matmul_intermediate_pad_rf_local[v__2.s7] = v__3.s7; int8 v__4 = (int8)((1)+(4*0), (1)+(4*1), (1)+(4*2), (1)+(4*3), (1)+(4*4), (1)+(4*5), (1)+(4*6), (1)+(4*7)); float8 v__5 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__4.s0],NT_matmul_intermediate_pad_rf_local[v__4.s1],NT_matmul_intermediate_pad_rf_local[v__4.s2],NT_matmul_intermediate_pad_rf_local[v__4.s3],NT_matmul_intermediate_pad_rf_local[v__4.s4],NT_matmul_intermediate_pad_rf_local[v__4.s5],NT_matmul_intermediate_pad_rf_local[v__4.s6],NT_matmul_intermediate_pad_rf_local[v__4.s7])) + ((convert_float8(vload8(0, layer_norm99 + ((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8))))) * (convert_float8(vload8(0, dequantize_local + 8)))); NT_matmul_intermediate_pad_rf_local[v__4.s0] = v__5.s0; NT_matmul_intermediate_pad_rf_local[v__4.s1] = v__5.s1; NT_matmul_intermediate_pad_rf_local[v__4.s2] = v__5.s2; NT_matmul_intermediate_pad_rf_local[v__4.s3] = v__5.s3; NT_matmul_intermediate_pad_rf_local[v__4.s4] = v__5.s4; NT_matmul_intermediate_pad_rf_local[v__4.s5] = v__5.s5; NT_matmul_intermediate_pad_rf_local[v__4.s6] = v__5.s6; NT_matmul_intermediate_pad_rf_local[v__4.s7] = v__5.s7; } if ((((convert_int(get_group_id(1))) * 2) + 1) < seq_len) { int8 v__6 = (int8)((2)+(4*0), (2)+(4*1), (2)+(4*2), (2)+(4*3), (2)+(4*4), (2)+(4*5), (2)+(4*6), (2)+(4*7)); float8 v__7 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__6.s0],NT_matmul_intermediate_pad_rf_local[v__6.s1],NT_matmul_intermediate_pad_rf_local[v__6.s2],NT_matmul_intermediate_pad_rf_local[v__6.s3],NT_matmul_intermediate_pad_rf_local[v__6.s4],NT_matmul_intermediate_pad_rf_local[v__6.s5],NT_matmul_intermediate_pad_rf_local[v__6.s6],NT_matmul_intermediate_pad_rf_local[v__6.s7])) + ((convert_float8(vload8(0, layer_norm99 + (((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)))) * (convert_float8(vload8(0, dequantize_local + 0)))); NT_matmul_intermediate_pad_rf_local[v__6.s0] = v__7.s0; NT_matmul_intermediate_pad_rf_local[v__6.s1] = v__7.s1; NT_matmul_intermediate_pad_rf_local[v__6.s2] = v__7.s2; NT_matmul_intermediate_pad_rf_local[v__6.s3] = v__7.s3; NT_matmul_intermediate_pad_rf_local[v__6.s4] = v__7.s4; NT_matmul_intermediate_pad_rf_local[v__6.s5] = v__7.s5; NT_matmul_intermediate_pad_rf_local[v__6.s6] = v__7.s6; NT_matmul_intermediate_pad_rf_local[v__6.s7] = v__7.s7; int8 v__8 = (int8)((3)+(4*0), (3)+(4*1), (3)+(4*2), (3)+(4*3), (3)+(4*4), (3)+(4*5), (3)+(4*6), (3)+(4*7)); float8 v__9 = ((float8)(NT_matmul_intermediate_pad_rf_local[v__8.s0],NT_matmul_intermediate_pad_rf_local[v__8.s1],NT_matmul_intermediate_pad_rf_local[v__8.s2],NT_matmul_intermediate_pad_rf_local[v__8.s3],NT_matmul_intermediate_pad_rf_local[v__8.s4],NT_matmul_intermediate_pad_rf_local[v__8.s5],NT_matmul_intermediate_pad_rf_local[v__8.s6],NT_matmul_intermediate_pad_rf_local[v__8.s7])) + ((convert_float8(vload8(0, layer_norm99 + (((((convert_int(get_group_id(1))) * 4096) + (ax2_fused_0 * 256)) + ((convert_int(get_local_id(1))) * 8)) + 2048)))) * (convert_float8(vload8(0, dequantize_local + 8)))); NT_matmul_intermediate_pad_rf_local[v__8.s0] = v__9.s0; NT_matmul_intermediate_pad_rf_local[v__8.s1] = v__9.s1; NT_matmul_intermediate_pad_rf_local[v__8.s2] = v__9.s2; NT_matmul_intermediate_pad_rf_local[v__8.s3] = v__9.s3; NT_matmul_intermediate_pad_rf_local[v__8.s4] = v__9.s4; NT_matmul_intermediate_pad_rf_local[v__8.s5] = v__9.s5; NT_matmul_intermediate_pad_rf_local[v__8.s6] = v__9.s6; NT_matmul_intermediate_pad_rf_local[v__8.s7] = v__9.s7; } } for (int ax3_fused_2_0 = 0; ax3_fused_2_0 < 1; ++ax3_fused_2_0) { for (int ax2 = 0; ax2 < 2; ++ax2) { vstore2(((float2)(0.000000e+00f, 0.000000e+00f)), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + (ax2 * 2))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 4))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 8))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 12))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 16))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 20))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 24))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); vstore2((vload2(0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)) + vload2(0, NT_matmul_intermediate_pad_rf_local + ((ax2 * 2) + 28))), 0, NT_matmul_intermediate_pad_rf_local_1 + (ax2 * 2)); } } for (int ax2_fused_2 = 0; ax2_fused_2 < 2; ++ax2_fused_2) { for (int ax1 = 0; ax1 < 2; ++ax1) { barrier(CLK_LOCAL_MEM_FENCE); ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = NT_matmul_intermediate_pad_rf_local_1[((ax1 * 2) + ax2_fused_2)]; barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 16) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 16)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 8) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 8)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 4) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 4)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 2) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 2)]); } barrier(CLK_LOCAL_MEM_FENCE); if ((convert_int(get_local_id(1))) < 1) { ((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] = (((volatile __local float*)red_buf0)[(((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1))))] + ((volatile __local float*)red_buf0)[((((convert_int(get_local_id(0))) * 32) + (convert_int(get_local_id(1)))) + 1)]); } barrier(CLK_LOCAL_MEM_FENCE); NT_matmul_intermediate_pad_local[((ax1 * 2) + ax2_fused_2)] = ((volatile __local float*)red_buf0)[((convert_int(get_local_id(0))) * 32)]; } } for (int ax0 = 0; ax0 < 2; ++ax0) { for (int ax1_fused_2 = 0; ax1_fused_2 < 2; ++ax1_fused_2) { if (((bool)(((bool)((convert_int(get_local_id(1))) == 0)) && ((bool)(((bool)(((convert_int(get_group_id(1))) - ((seq_len + 1) >> 1)) < 0)) || ((bool)((convert_int(get_group_id(1))) == 0)))))) && ((bool)((((convert_int(get_group_id(1))) * 2) + ax0) < seq_len))) { compute[((((((convert_int(get_group_id(1))) * 16384) + (ax0 * 8192)) + ((convert_int(get_group_id(0))) * 4)) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)] = (convert_half(((NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]) * (5.000000e-01f + (erf(((NT_matmul_intermediate_pad_local[((ax0 * 2) + ax1_fused_2)] + gpt_neox_layers_0_mlp_dense_h_to_4h_bias3[((((convert_int(get_group_id(0))) * 4) + ((convert_int(get_local_id(0))) * 2)) + ax1_fused_2)]) * 7.071068e-01f)) * 5.000000e-01f))))); } } } }