kat0012 commited on
Commit
6609e2b
·
verified ·
1 Parent(s): 3ad1e93

Parakon runtime: llama.cpp fork with fused 1-bit CUDA/Metal/CPU kernels (part 6)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. ggml/src/ggml-sycl/quants.hpp +204 -0
  2. ggml/src/ggml-sycl/repeat_back.cpp +76 -0
  3. ggml/src/ggml-sycl/repeat_back.hpp +8 -0
  4. ggml/src/ggml-sycl/roll.cpp +122 -0
  5. ggml/src/ggml-sycl/roll.hpp +20 -0
  6. ggml/src/ggml-sycl/rope.cpp +651 -0
  7. ggml/src/ggml-sycl/rope.hpp +26 -0
  8. ggml/src/ggml-sycl/set.cpp +73 -0
  9. ggml/src/ggml-sycl/set.hpp +5 -0
  10. ggml/src/ggml-sycl/set_rows.cpp +577 -0
  11. ggml/src/ggml-sycl/set_rows.hpp +8 -0
  12. ggml/src/ggml-sycl/softmax.cpp +425 -0
  13. ggml/src/ggml-sycl/softmax.hpp +24 -0
  14. ggml/src/ggml-sycl/solve_tri.cpp +172 -0
  15. ggml/src/ggml-sycl/solve_tri.hpp +8 -0
  16. ggml/src/ggml-sycl/ssm_conv.cpp +136 -0
  17. ggml/src/ggml-sycl/ssm_conv.hpp +5 -0
  18. ggml/src/ggml-sycl/ssm_scan.cpp +170 -0
  19. ggml/src/ggml-sycl/ssm_scan.hpp +5 -0
  20. ggml/src/ggml-sycl/sycl_hw.cpp +67 -0
  21. ggml/src/ggml-sycl/sycl_hw.hpp +38 -0
  22. ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq112-dv112.cpp +5 -0
  23. ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq128-dv128.cpp +5 -0
  24. ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq256-dv256.cpp +5 -0
  25. ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq40-dv40.cpp +5 -0
  26. ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq512-dv512.cpp +6 -0
  27. ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq576-dv512.cpp +5 -0
  28. ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq64-dv64.cpp +5 -0
  29. ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq72-dv72.cpp +5 -0
  30. ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq80-dv80.cpp +5 -0
  31. ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq96-dv96.cpp +5 -0
  32. ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-f16.cpp +8 -0
  33. ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_0.cpp +8 -0
  34. ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_1.cpp +8 -0
  35. ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_0.cpp +8 -0
  36. ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_1.cpp +8 -0
  37. ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q8_0.cpp +8 -0
  38. ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-f16.cpp +8 -0
  39. ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_0.cpp +8 -0
  40. ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_1.cpp +8 -0
  41. ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_0.cpp +8 -0
  42. ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_1.cpp +8 -0
  43. ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q8_0.cpp +8 -0
  44. ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-f16.cpp +8 -0
  45. ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_0.cpp +8 -0
  46. ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_1.cpp +8 -0
  47. ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_0.cpp +8 -0
  48. ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_1.cpp +8 -0
  49. ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q8_0.cpp +8 -0
  50. ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-f16.cpp +8 -0
ggml/src/ggml-sycl/quants.hpp ADDED
@@ -0,0 +1,204 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //
2
+ // MIT license
3
+ // Copyright (C) 2025 Codeplay Software Ltd.
4
+ // Copyright (C) 2025 Intel Corporation
5
+ // SPDX-License-Identifier: MIT
6
+ //
7
+
8
+ //
9
+ // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
10
+ // See https://llvm.org/LICENSE.txt for license information.
11
+ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
12
+ //
13
+
14
+ #ifndef GGML_SYCL_QUANTS_HPP
15
+ #define GGML_SYCL_QUANTS_HPP
16
+
17
+ #include <utility>
18
+
19
+ #include "ggml-common.h"
20
+ #include "ggml.h"
21
+
22
+ namespace ggml_sycl_reordered {
23
+
24
+ // The reordered block moves quants (qs) and scales(d) to two
25
+ // uniform regions of memory that is contiguous in the same tensor.
26
+ // What this means is that instead of having:
27
+ // [d0, qs0] [d1, qs1] [d2, qs2] ... [dN, qsN]
28
+ // We have:
29
+ // [qs0, qs1, qs2, ..., qsN] [d0, d1, d2, ..., dN]
30
+ //
31
+ // Notes: out-of-bounds qs will run into d values
32
+ // Alignment relies on the allocated size of qs
33
+
34
+ template <ggml_type type> struct block_q_t;
35
+
36
+ // qk number of weights / quants in a block
37
+ // qr number of weights in a byte (described as 'before dequantization')
38
+ // for quantization types that has low and high bits split, qr is calculated with
39
+ // using the lower bits, e.g for Q6 quants QR6 is 2
40
+ // qi number of 32 bit integers needed to represent all the quants from a block (`qs` field)
41
+ // See ggml-common.h to see how these are calculated
42
+ template <> struct block_q_t<GGML_TYPE_Q4_0> {
43
+ struct traits {
44
+ static constexpr uint32_t qk = QK4_0;
45
+ static constexpr uint32_t qi = QI4_0;
46
+ static constexpr uint32_t qr = QR4_0;
47
+ static constexpr uint32_t vdr_mmvq = 2;
48
+ };
49
+
50
+ static constexpr std::pair<int, int> get_block_offset(const int block_index, const int /* nblocks */) {
51
+ return { block_index * (QK4_0 / QR4_0), 0 };
52
+ }
53
+
54
+ static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) {
55
+ return { (ncols / QR4_0 * nrows) + block_index * sizeof(ggml_half), 0 };
56
+ }
57
+
58
+ static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }
59
+ };
60
+
61
+ template <> struct block_q_t<GGML_TYPE_Q2_K> {
62
+ struct traits {
63
+ static constexpr uint32_t qk = QK_K;
64
+ static constexpr uint32_t qi = QI2_K;
65
+ static constexpr uint32_t qr = QR2_K;
66
+ static constexpr uint32_t vdr_mmvq = 1;
67
+ };
68
+
69
+ // Reordered layout: [qs (QK_K/4 per block)] [scales (QK_K/16 per block)] [dm]
70
+ static constexpr std::pair<int, int> get_block_offset(const int block_index, const int /* n_blocks */) {
71
+ return { block_index * (QK_K / 4), 0 };
72
+ }
73
+
74
+ static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) {
75
+ auto nblocks = (nrows * (ncols / QK_K));
76
+ auto total_qs_bytes = nblocks * (QK_K / 4);
77
+ return { total_qs_bytes + block_index * (QK_K / 16),
78
+ total_qs_bytes + nblocks * (QK_K / 16) + block_index * sizeof(ggml_half2) };
79
+ }
80
+
81
+ static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }
82
+ };
83
+
84
+ template <> struct block_q_t<GGML_TYPE_Q3_K> {
85
+ struct traits {
86
+ static constexpr uint32_t qk = QK_K;
87
+ static constexpr uint32_t qi = QI3_K;
88
+ static constexpr uint32_t qr = QR3_K;
89
+ static constexpr uint32_t vdr_mmvq = 1;
90
+ };
91
+
92
+ // Reordered layout: [qs (QK_K/4 per block)] [hmask (QK_K/8 per block)] [scales] [d]
93
+ static constexpr std::pair<int, int> get_block_offset(const int block_index, const int n_blocks) {
94
+ auto qs_offset = block_index * (QK_K / 4);
95
+ auto hmask_offset = n_blocks * (QK_K / 4) + block_index * (QK_K / 8);
96
+ return { qs_offset, hmask_offset };
97
+ }
98
+
99
+ static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) {
100
+ auto nblocks = (nrows * (ncols / QK_K));
101
+ auto total_qs_bytes = nblocks * (QK_K / 4) + nblocks * (QK_K / 8);
102
+ return { total_qs_bytes + block_index * 12,
103
+ total_qs_bytes + nblocks * 12 + block_index * sizeof(ggml_half) };
104
+ }
105
+
106
+ static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }
107
+ };
108
+
109
+ template <> struct block_q_t<GGML_TYPE_Q4_K> {
110
+ struct traits {
111
+ static constexpr uint32_t qk = QK_K;
112
+ static constexpr uint32_t qi = QI4_K;
113
+ static constexpr uint32_t qr = QR4_K;
114
+ static constexpr uint32_t vdr_mmvq = 2;
115
+ };
116
+
117
+ static constexpr std::pair<int, int> get_block_offset(const int block_index, const int /* nblocks */) {
118
+ return { block_index * (traits::qk / traits::qr), 0 };
119
+ }
120
+
121
+ static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) {
122
+ auto nblocks = (nrows * (ncols / QK_K));
123
+ return { nblocks * (QK_K / 2) + (block_index * K_SCALE_SIZE),
124
+ (nblocks * QK_K / 2) + (nblocks * K_SCALE_SIZE) + (block_index * sizeof(ggml_half2)) };
125
+ }
126
+
127
+ static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }
128
+ };
129
+
130
+ template <> struct block_q_t<GGML_TYPE_Q5_K> {
131
+ struct traits {
132
+ static constexpr uint32_t qk = QK_K;
133
+ static constexpr uint32_t qi = QI5_K;
134
+ static constexpr uint32_t qr = QR5_K;
135
+ static constexpr uint32_t vdr_mmvq = 2;
136
+ };
137
+
138
+ // Reordered layout: [qs (QK_K/2 per block)] [qh (QK_K/8 per block)] [scales] [dm]
139
+ static constexpr std::pair<int, int> get_block_offset(const int block_index, const int n_blocks) {
140
+ auto qs_offset = block_index * (QK_K / 2);
141
+ auto qh_offset = n_blocks * (QK_K / 2) + block_index * (QK_K / 8);
142
+ return { qs_offset, qh_offset };
143
+ }
144
+
145
+ static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) {
146
+ auto nblocks = (nrows * (ncols / QK_K));
147
+ auto total_qs_bytes = nblocks * (QK_K / 2) + nblocks * (QK_K / 8);
148
+ return { total_qs_bytes + block_index * K_SCALE_SIZE,
149
+ total_qs_bytes + nblocks * K_SCALE_SIZE + block_index * sizeof(ggml_half2) };
150
+ }
151
+
152
+ static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }
153
+ };
154
+
155
+ template <> struct block_q_t<GGML_TYPE_Q6_K> {
156
+ struct traits {
157
+ static constexpr uint32_t qk = QK_K;
158
+ static constexpr uint32_t qi = QI6_K;
159
+ static constexpr uint32_t qr = QR6_K;
160
+ static constexpr uint32_t vdr_mmvq = 1;
161
+ };
162
+
163
+ static constexpr std::pair<int, int> get_block_offset(const int block_index, const int n_blocks) {
164
+ auto low_bits_index = block_index * (QK_K / QR6_K);
165
+ // the index of high bits it's after all low bits
166
+ auto high_bits_index = n_blocks * (QK_K / 2) + (block_index * (QK_K / 4));
167
+ return { low_bits_index, high_bits_index };
168
+ }
169
+
170
+ static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) {
171
+ auto nblocks = (nrows * (ncols / QK_K));
172
+ auto total_qs_bytes = nblocks * (QK_K / 2) + nblocks * (QK_K / 4);
173
+ auto block_scales = total_qs_bytes + block_index * (QK_K / 16);
174
+ auto sb_scale = total_qs_bytes + nblocks * (QK_K / 16) + block_index * sizeof(ggml_half);
175
+ return { block_scales, sb_scale };
176
+ }
177
+
178
+ static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }
179
+ };
180
+
181
+ template <> struct block_q_t<GGML_TYPE_Q8_0> {
182
+ struct traits {
183
+ static constexpr uint32_t qk = QK8_0; // 32
184
+ static constexpr uint32_t qi = QI8_0; // 8
185
+ static constexpr uint32_t qr = QR8_0; // 1
186
+ static constexpr uint32_t vdr_mmvq = 4;
187
+ };
188
+
189
+ // Q8_0 reorder layout: [qs0|qs1|...|qsN][d0|d1|...|dN]
190
+ // Each block has 32 int8 weights (32 bytes) followed by all scales
191
+ static constexpr std::pair<int, int> get_block_offset(const int block_index, const int /* nblocks */) {
192
+ return { block_index * QK8_0, 0 };
193
+ }
194
+
195
+ static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) {
196
+ return { (ncols * nrows) + block_index * sizeof(ggml_half), 0 };
197
+ }
198
+
199
+ static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; } // 1
200
+ };
201
+
202
+ } // namespace ggml_sycl_reordered
203
+
204
+ #endif // GGML_SYCL_QUANTS_HPP
ggml/src/ggml-sycl/repeat_back.cpp ADDED
@@ -0,0 +1,76 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #include "repeat_back.hpp"
2
+
3
+ #include "common.hpp"
4
+
5
+ #define GGML_ASSERT_TENSOR_FITS_INT(t) \
6
+ GGML_ASSERT((t)->ne[0] < INT_MAX && (t)->ne[1] < INT_MAX && (t)->ne[2] < INT_MAX && (t)->ne[3] < INT_MAX)
7
+
8
+ void ggml_sycl_op_repeat_back(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
9
+ GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);
10
+ GGML_ASSERT(dst->type == GGML_TYPE_F32);
11
+
12
+ const float * src0_dd = (const float *) dst->src[0]->data;
13
+ float * dst_dd = (float *) dst->data;
14
+
15
+ GGML_ASSERT_TENSOR_FITS_INT(dst);
16
+ GGML_ASSERT_TENSOR_FITS_INT(dst->src[0]);
17
+
18
+ const int ne0 = dst->ne[0], ne1 = dst->ne[1], ne2 = dst->ne[2], ne3 = dst->ne[3];
19
+ const int ne00 = dst->src[0]->ne[0], ne01 = dst->src[0]->ne[1], ne02 = dst->src[0]->ne[2],
20
+ ne03 = dst->src[0]->ne[3];
21
+
22
+ const int nr0 = ne00 / ne0;
23
+ const int nr1 = ne01 / ne1;
24
+ const int nr2 = ne02 / ne2;
25
+ const int nr3 = ne03 / ne3;
26
+
27
+ const int nb0 = dst->src[0]->nb[0];
28
+ const int nb1 = dst->src[0]->nb[1];
29
+ const int nb2 = dst->src[0]->nb[2];
30
+ const int nb3 = dst->src[0]->nb[3];
31
+
32
+ const char * base = (const char *) src0_dd;
33
+
34
+ const size_t total = (size_t) ne0 * ne1 * ne2 * ne3;
35
+ constexpr int BLOCK_SIZE = 256;
36
+ const int num_blocks = (total + BLOCK_SIZE - 1) / BLOCK_SIZE;
37
+
38
+ const float inv_ne0 = 1.0f / ne0;
39
+ const float inv_ne_01 = 1.0f / (ne0 * ne1);
40
+ const float inv_ne_012 = 1.0f / (ne0 * ne1 * ne2);
41
+ const int repeat_count = nr0 * nr1 * nr2 * nr3;
42
+
43
+ queue_ptr stream = ctx.stream();
44
+
45
+ stream->parallel_for(
46
+ sycl::nd_range<1>(sycl::range<1>(num_blocks * BLOCK_SIZE), sycl::range<1>(BLOCK_SIZE)),
47
+ [=](sycl::nd_item<1> item_ct1) {
48
+ const size_t i = item_ct1.get_global_linear_id();
49
+ if (i >= total) {
50
+ return;
51
+ }
52
+
53
+ const int i3 = (int) (i * inv_ne_012);
54
+ const int i2 = (int) (i * inv_ne_01) - i3 * ne2;
55
+ const int i1 = (int) (i * inv_ne0) - (int) (i * inv_ne_01) * ne1;
56
+ const int i0 = i - (int) (i * inv_ne0) * ne0;
57
+
58
+ int j0 = 0, j1 = 0, j2 = 0, j3 = 0;
59
+ float acc = 0.0f;
60
+
61
+ for (int j = 0; j < repeat_count; ++j) {
62
+ const float * ptr = (const float *) (base + (i0 + j0 * ne0) * nb0 + (i1 + j1 * ne1) * nb1 +
63
+ (i2 + j2 * ne2) * nb2 + (i3 + j3 * ne3) * nb3);
64
+ acc += *ptr;
65
+
66
+ int carry = (++j0 >= nr0);
67
+ j0 -= carry * nr0;
68
+ carry = (carry && (++j1 >= nr1));
69
+ j1 -= carry * nr1;
70
+ carry = (carry && (++j2 >= nr2));
71
+ j2 -= carry * nr2;
72
+ j3 += carry;
73
+ }
74
+ dst_dd[i] = acc;
75
+ });
76
+ }
ggml/src/ggml-sycl/repeat_back.hpp ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ #ifndef GGML_SYCL_REPEAT_BACK_HPP
2
+ #define GGML_SYCL_REPEAT_BACK_HPP
3
+
4
+ #include "common.hpp"
5
+
6
+ void ggml_sycl_op_repeat_back(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
7
+
8
+ #endif // GGML_SYCL_REPEAT_BACK_HPP
ggml/src/ggml-sycl/roll.cpp ADDED
@@ -0,0 +1,122 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #include "roll.hpp"
2
+ #include "common.hpp"
3
+
4
+ using namespace sycl;
5
+
6
+ static inline int wrap_add(int i, int shift, int n) {
7
+
8
+ int s = i + shift;
9
+ return (s >= n) ? (s - n) : s;
10
+ }
11
+
12
+ static void kernel_roll_fused_i0_i1(
13
+ queue &q,
14
+ const float *src_d,
15
+ float *dst_d,
16
+ int ne0, int ne1, int ne2, int ne3,
17
+ int sh0, int sh1, int sh2, int sh3)
18
+ {
19
+ if (ne0 == 0 || ne1 == 0 || ne2 == 0 || ne3 == 0) return;
20
+
21
+
22
+ const int stride1 = ne0;
23
+ const int stride2 = ne0 * ne1;
24
+ const int stride3 = ne0 * ne1 * ne2;
25
+
26
+
27
+ const int shNe0 = (ne0 - sh0) % ne0;
28
+ const int shNe1 = (ne1 - sh1) % ne1;
29
+ const int shNe2 = (ne2 - sh2) % ne2;
30
+ const int shNe3 = (ne3 - sh3) % ne3;
31
+
32
+
33
+ const size_t g0 = (size_t) ne3;
34
+ const size_t g1 = (size_t) ne2;
35
+ const size_t g2 = (size_t) (ne1 * ne0);
36
+
37
+ const range<3> global{ g0, g1, g2 };
38
+
39
+ q.submit([&](handler &h) {
40
+ h.parallel_for(global, [=](id<3> idx) {
41
+ const int i3 = (int) idx[0];
42
+ const int i2 = (int) idx[1];
43
+
44
+ const int fused = (int) idx[2];
45
+ const int i1 = fused / ne0;
46
+ const int i0 = fused - i1 * ne0; // fused % ne0
47
+
48
+
49
+ const int idx_dst = i0
50
+ + i1 * stride1
51
+ + i2 * stride2
52
+ + i3 * stride3;
53
+
54
+
55
+ const int s0 = wrap_add(i0, shNe0, ne0);
56
+ const int s1 = wrap_add(i1, shNe1, ne1);
57
+ const int s2 = wrap_add(i2, shNe2, ne2);
58
+ const int s3 = wrap_add(i3, shNe3, ne3);
59
+
60
+ const int idx_src = s0
61
+ + s1 * stride1
62
+ + s2 * stride2
63
+ + s3 * stride3;
64
+
65
+ dst_d[idx_dst] = src_d[idx_src];
66
+ });
67
+ });
68
+ }
69
+
70
+ void ggml_sycl_roll(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {
71
+ GGML_ASSERT(dst->type == GGML_TYPE_F32);
72
+
73
+ const ggml_tensor *src = dst->src[0];
74
+ GGML_ASSERT(src && src->type == GGML_TYPE_F32);
75
+
76
+ const int ne0 = (int) dst->ne[0];
77
+ const int ne1 = (int) dst->ne[1];
78
+ const int ne2 = (int) dst->ne[2];
79
+ const int ne3 = (int) dst->ne[3];
80
+
81
+ const int32_t *params = (const int32_t *) dst->op_params;
82
+ int shift0 = params[0];
83
+ int shift1 = params[1];
84
+ int shift2 = params[2];
85
+ int shift3 = params[3];
86
+
87
+
88
+ if ((shift0 | shift1 | shift2 | shift3) == 0) {
89
+ const size_t nb = ggml_nbytes(src);
90
+ queue *q = ctx.stream();
91
+ SYCL_CHECK(CHECK_TRY_ERROR(q->memcpy(dst->data, src->data, nb)));
92
+ return;
93
+ }
94
+
95
+ auto norm = [](int sh, int n) -> int {
96
+ if (n <= 0) return 0;
97
+ sh %= n;
98
+ if (sh < 0) sh += n;
99
+ return sh;
100
+ };
101
+ shift0 = norm(shift0, ne0);
102
+ shift1 = norm(shift1, ne1);
103
+ shift2 = norm(shift2, ne2);
104
+ shift3 = norm(shift3, ne3);
105
+
106
+ try {
107
+ queue *q = ctx.stream();
108
+
109
+ const float *src_d = (const float *) src->data;
110
+ float *dst_d = (float *) dst->data;
111
+ GGML_ASSERT(src_d && dst_d);
112
+
113
+ kernel_roll_fused_i0_i1(
114
+ *q, src_d, dst_d,
115
+ ne0, ne1, ne2, ne3,
116
+ shift0, shift1, shift2, shift3
117
+ );
118
+ } catch (const std::exception &e) {
119
+ std::fprintf(stderr, "[SYCL-ROLL] ERROR: %s\n", e.what());
120
+ throw;
121
+ }
122
+ }
ggml/src/ggml-sycl/roll.hpp ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //
2
+ // MIT license
3
+ // Copyright (C) 2024 Intel Corporation
4
+ // SPDX-License-Identifier: MIT
5
+ //
6
+
7
+ //
8
+ // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
9
+ // See https://llvm.org/LICENSE.txt for license information.
10
+ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
11
+ //
12
+
13
+ #ifndef GGML_SYCL_ROLL_HPP
14
+ #define GGML_SYCL_ROLL_HPP
15
+
16
+ #include "common.hpp"
17
+
18
+ void ggml_sycl_roll(ggml_backend_sycl_context & ctx, ggml_tensor *dst);
19
+
20
+ #endif // GGML_SYCL_ROLL_HPP
ggml/src/ggml-sycl/rope.cpp ADDED
@@ -0,0 +1,651 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #include "rope.hpp"
2
+ #include "convert.hpp"
3
+ #include "ggml-sycl/common.hpp"
4
+ #include "ggml.h"
5
+
6
+ struct rope_corr_dims {
7
+ float v[2];
8
+ };
9
+
10
+ struct mrope_sections {
11
+ int v[4];
12
+ };
13
+
14
+ static float rope_yarn_ramp(const float low, const float high, const int i0) {
15
+ const float y = (i0 / 2 - low) / sycl::max(0.001f, high - low);
16
+ return 1.0f - sycl::min(1.0f, sycl::max(0.0f, y));
17
+ }
18
+
19
+ template <bool forward>
20
+ static void rope_yarn(const float theta_extrap, const float freq_scale,
21
+ const rope_corr_dims corr_dims, const int64_t i0,
22
+ const float ext_factor, float mscale, float &cos_theta,
23
+ float &sin_theta) {
24
+ float theta_interp = freq_scale * theta_extrap;
25
+ float theta = theta_interp;
26
+ if (ext_factor != 0.0f) {
27
+ float ramp_mix =
28
+ rope_yarn_ramp(corr_dims.v[0], corr_dims.v[1], i0) * ext_factor;
29
+ theta = theta_interp * (1 - ramp_mix) + theta_extrap * ramp_mix;
30
+
31
+ mscale *= 1.0f + 0.1f * sycl::log(1.0f / freq_scale);
32
+ }
33
+ cos_theta = sycl::cos(theta) * mscale;
34
+ sin_theta = sycl::sin(theta) * mscale;
35
+ if (!forward) {
36
+ sin_theta *= -1.0f;
37
+ }
38
+ }
39
+
40
+ template <bool forward, bool has_ff, typename T, typename D>
41
+ static void rope_norm(const T *x, D *dst, const int ne00, const int ne01,
42
+ const int ne02, const int s01, const int s02,
43
+ const int s03, const int s1, const int s2, const int s3,
44
+ const int n_dims, const int n_offs, const int32_t *pos,
45
+ const float freq_scale, const float ext_factor,
46
+ const float attn_factor, const rope_corr_dims corr_dims,
47
+ const float theta_scale, const float *freq_factors,
48
+ const int64_t *row_indices, const int set_rows_stride) {
49
+ auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
50
+ const int i0 = 2 * (item_ct1.get_local_range(1) * item_ct1.get_group(1) +
51
+ item_ct1.get_local_id(1));
52
+
53
+ if (i0 >= ne00) {
54
+ return;
55
+ }
56
+
57
+ const int row_dst = item_ct1.get_local_range(2) * item_ct1.get_group(2) +
58
+ item_ct1.get_local_id(2);
59
+
60
+ const uint32_t i3 = row_dst / (ne01 * ne02);
61
+ const uint32_t i2 = (row_dst - i3 * ne01 * ne02) / ne01;
62
+ const uint32_t i1 = row_dst - i3 * ne01 * ne02 - i2 * ne01;
63
+
64
+ int idst = i0 + i1 * s1 + i2 * s2 + i3 * s3;
65
+ const int ix = i0 + i1 * s01 + i2 * s02 + i3 * s03;
66
+
67
+ if (set_rows_stride != 0) {
68
+ idst = i1 * s1 + i0;
69
+ idst += row_indices[i2] * set_rows_stride;
70
+ }
71
+
72
+ const auto &store_coaelsced = [&](float x0, float x1) {
73
+ if constexpr (std::is_same_v<float, D>) {
74
+ sycl::float2 v = sycl::float2(x0, x1);
75
+ ggml_sycl_memcpy_1<8>(dst + idst, &v);
76
+ } else if constexpr (std::is_same_v<sycl::half, D>) {
77
+ sycl::half2 v = sycl::half2(x0, x1);
78
+ ggml_sycl_memcpy_1<4>(dst + idst, &v);
79
+ }
80
+ };
81
+ if (i0 < n_offs || i0 >= n_offs + n_dims) {
82
+ store_coaelsced(x[ix + 0], x[ix + 1]);
83
+ return;
84
+ }
85
+
86
+ const int iw = i0 - n_offs; // relative idx
87
+
88
+ const float theta_base = pos[i2] * dpct::pow(theta_scale, iw / 2.0f);
89
+
90
+ const float freq_factor = has_ff ? freq_factors[iw / 2] : 1.0f;
91
+
92
+ float cos_theta;
93
+ float sin_theta;
94
+
95
+ rope_yarn<forward>(theta_base / freq_factor, freq_scale, corr_dims, iw,
96
+ ext_factor, attn_factor, cos_theta, sin_theta);
97
+
98
+ const float x0 = x[ix + 0];
99
+ const float x1 = x[ix + 1];
100
+
101
+ store_coaelsced(x0 * cos_theta - x1 * sin_theta,
102
+ x0 * sin_theta + x1 * cos_theta);
103
+ }
104
+
105
+ template <bool forward, bool has_ff, typename T, typename D>
106
+ static void rope_neox(const T *x, D *dst, const int ne00, const int ne01,
107
+ const int ne02, const int s01, const int s02,
108
+ const int s03, const int s1, const int s2, const int s3,
109
+ const int n_dims, const int n_offs, const int32_t *pos,
110
+ const float freq_scale, const float ext_factor,
111
+ const float attn_factor, const rope_corr_dims corr_dims,
112
+ const float theta_scale, const float *freq_factors,
113
+ const int64_t *row_indices, const int set_rows_stride) {
114
+ auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
115
+ const int i0 = 2 * (item_ct1.get_local_range(1) * item_ct1.get_group(1) +
116
+ item_ct1.get_local_id(1));
117
+
118
+ if (i0 >= ne00) {
119
+ return;
120
+ }
121
+
122
+ const int row_dst = item_ct1.get_local_range(2) * item_ct1.get_group(2) +
123
+ item_ct1.get_local_id(2);
124
+
125
+ const uint32_t i3 = row_dst / (ne01 * ne02);
126
+ const uint32_t i2 = (row_dst - i3 * ne01 * ne02) / ne01;
127
+ const uint32_t i1 = row_dst - i3 * ne01 * ne02 - i2 * ne01;
128
+
129
+ int idst = i0 / 2 + i1 * s1 + i2 * s2 + i3 * s3;
130
+ const int ix = i0 / 2 + i1 * s01 + i2 * s02 + i3 * s03;
131
+
132
+ if (set_rows_stride != 0) {
133
+ idst = i1 * s1 + i0 / 2;
134
+ idst += row_indices[i2] * set_rows_stride;
135
+ }
136
+
137
+ if (i0 < n_offs || i0 >= n_offs + n_dims) {
138
+ dst[idst + i0 / 2 + 0] = ggml_sycl_cast<D>(x[ix + i0 / 2 + 0]);
139
+ dst[idst + i0 / 2 + 1] = ggml_sycl_cast<D>(x[ix + i0 / 2 + 1]);
140
+
141
+ return;
142
+ }
143
+
144
+ const int iw = i0 - n_offs; // relative idx
145
+
146
+ const float theta_base = pos[i2] * dpct::pow(theta_scale, iw / 2.0f);
147
+
148
+ const float freq_factor = has_ff ? freq_factors[iw / 2] : 1.0f;
149
+
150
+ float cos_theta;
151
+ float sin_theta;
152
+
153
+ rope_yarn<forward>(theta_base / freq_factor, freq_scale, corr_dims, iw,
154
+ ext_factor, attn_factor, cos_theta, sin_theta);
155
+
156
+ // idst/ix point at channel i0/2; the first channel of the rotated pair is n_offs + iw/2 = i0/2 + n_offs/2
157
+ const float x0 = x[ix + n_offs / 2 + 0];
158
+ const float x1 = x[ix + n_offs / 2 + n_dims / 2];
159
+
160
+ dst[idst + n_offs / 2 + 0] = ggml_sycl_cast<D>(x0 * cos_theta - x1 * sin_theta);
161
+ dst[idst + n_offs / 2 + n_dims / 2] = ggml_sycl_cast<D>(x0 * sin_theta + x1 * cos_theta);
162
+ }
163
+
164
+ template <bool forward, bool has_ff, typename T>
165
+ static void rope_multi(const T *x, T *dst, const int ne00, const int ne01,
166
+ const int ne02, const int s01, const int s02,
167
+ const int s03, const int s1, const int s2, const int s3,
168
+ const int n_dims, const int n_offs, const int32_t *pos,
169
+ const float freq_scale, const float ext_factor,
170
+ const float attn_factor, const rope_corr_dims corr_dims,
171
+ const float theta_scale, const float *freq_factors,
172
+ const mrope_sections sections, const bool is_imrope) {
173
+ auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
174
+ const int i0 = 2 * (item_ct1.get_local_range(1) * item_ct1.get_group(1) +
175
+ item_ct1.get_local_id(1));
176
+
177
+ if (i0 >= ne00) {
178
+ return;
179
+ }
180
+
181
+ const int row_dst = item_ct1.get_local_range(2) * item_ct1.get_group(2) +
182
+ item_ct1.get_local_id(2);
183
+
184
+ const uint32_t i3 = row_dst / (ne01 * ne02);
185
+ const uint32_t i2 = (row_dst - i3 * ne01 * ne02) / ne01;
186
+ const uint32_t i1 = row_dst - i3 * ne01 * ne02 - i2 * ne01;
187
+
188
+ int idst = i0 / 2 + i1 * s1 + i2 * s2 + i3 * s3;
189
+ const int ix = i0 / 2 + i1 * s01 + i2 * s02 + i3 * s03;
190
+
191
+ if (i0 < n_offs || i0 >= n_offs + n_dims) {
192
+ dst[idst + i0 / 2 + 0] = x[ix + i0 / 2 + 0];
193
+ dst[idst + i0 / 2 + 1] = x[ix + i0 / 2 + 1];
194
+
195
+ return;
196
+ }
197
+
198
+ const int iw = i0 - n_offs; // relative idx
199
+
200
+ const int sect_dims =
201
+ sections.v[0] + sections.v[1] + sections.v[2] + sections.v[3];
202
+ const int sec_w = sections.v[1] + sections.v[0];
203
+ const int sector = (iw / 2) % sect_dims;
204
+
205
+ float theta_base = 0.0;
206
+ if (is_imrope) {
207
+ if (sector % 3 == 1 && sector < 3 * sections.v[1]) { // h
208
+ theta_base = pos[i2 + ne02 * 1] * dpct::pow(theta_scale, iw / 2.0f);
209
+ } else if (sector % 3 == 2 && sector < 3 * sections.v[2]) { // w
210
+ theta_base = pos[i2 + ne02 * 2] * dpct::pow(theta_scale, iw / 2.0f);
211
+ } else if (sector % 3 == 0 && sector < 3 * sections.v[0]) { // t
212
+ theta_base = pos[i2] * dpct::pow(theta_scale, iw / 2.0f);
213
+ } else {
214
+ theta_base = pos[i2 + ne02 * 3] * dpct::pow(theta_scale, iw / 2.0f);
215
+ }
216
+ } else {
217
+ if (sector < sections.v[0]) {
218
+ theta_base = pos[i2] * dpct::pow(theta_scale, iw / 2.0f);
219
+ } else if (sector >= sections.v[0] && sector < sec_w) {
220
+ theta_base = pos[i2 + ne02 * 1] * dpct::pow(theta_scale, iw / 2.0f);
221
+ } else if (sector >= sec_w && sector < sec_w + sections.v[2]) {
222
+ theta_base = pos[i2 + ne02 * 2] * dpct::pow(theta_scale, iw / 2.0f);
223
+ } else if (sector >= sec_w + sections.v[2]) {
224
+ theta_base = pos[i2 + ne02 * 3] * dpct::pow(theta_scale, iw / 2.0f);
225
+ }
226
+ }
227
+
228
+ const float freq_factor = has_ff ? freq_factors[iw / 2] : 1.0f;
229
+
230
+ float cos_theta;
231
+ float sin_theta;
232
+
233
+ rope_yarn<forward>(theta_base / freq_factor, freq_scale, corr_dims, iw,
234
+ ext_factor, attn_factor, cos_theta, sin_theta);
235
+
236
+ // idst/ix point at channel i0/2; the first channel of the rotated pair is n_offs + iw/2 = i0/2 + n_offs/2
237
+ const float x0 = x[ix + n_offs / 2 + 0];
238
+ const float x1 = x[ix + n_offs / 2 + n_dims / 2];
239
+
240
+ dst[idst + n_offs / 2 + 0] = x0 * cos_theta - x1 * sin_theta;
241
+ dst[idst + n_offs / 2 + n_dims / 2] = x0 * sin_theta + x1 * cos_theta;
242
+ }
243
+
244
+ template <bool forward, bool has_ff, typename T>
245
+ static void rope_vision(const T *x, T *dst, const int ne00, const int ne01,
246
+ const int ne02, const int s01, const int s02,
247
+ const int s03, const int s1, const int s2, const int s3,
248
+ const int n_dims, const int32_t *pos,
249
+ const float freq_scale, const float ext_factor,
250
+ const float attn_factor, const rope_corr_dims corr_dims,
251
+ const float theta_scale, const float *freq_factors,
252
+ const mrope_sections sections) {
253
+ auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
254
+ const int i0 = 2 * (item_ct1.get_local_range(1) * item_ct1.get_group(1) +
255
+ item_ct1.get_local_id(1));
256
+
257
+ if (i0 >= ne00) {
258
+ return;
259
+ }
260
+
261
+ const int row_dst = item_ct1.get_local_range(2) * item_ct1.get_group(2) +
262
+ item_ct1.get_local_id(2);
263
+
264
+ const uint32_t i3 = row_dst / (ne01 * ne02);
265
+ const uint32_t i2 = (row_dst - i3 * ne01 * ne02) / ne01;
266
+ const uint32_t i1 = row_dst - i3 * ne01 * ne02 - i2 * ne01;
267
+
268
+ int idst = i0 / 2 + i1 * s1 + i2 * s2 + i3 * s3;
269
+ const int ix = i0 / 2 + i1 * s01 + i2 * s02 + i3 * s03;
270
+
271
+ const int sect_dims = sections.v[0] + sections.v[1];
272
+ const int sec_w = sections.v[1] + sections.v[0];
273
+ const int sector = (i0 / 2) % sect_dims;
274
+
275
+ float theta_base = 0.0;
276
+ if (sector < sections.v[0]) {
277
+ const int p = sector;
278
+ theta_base = pos[i2] * dpct::pow(theta_scale, p);
279
+ } else if (sector >= sections.v[0] && sector < sec_w) {
280
+ const int p = sector - sections.v[0];
281
+ theta_base = pos[i2 + ne02] * dpct::pow(theta_scale, p);
282
+ }
283
+
284
+ const float freq_factor = has_ff ? freq_factors[i0 / 2] : 1.0f;
285
+
286
+ float cos_theta;
287
+ float sin_theta;
288
+
289
+ rope_yarn<forward>(theta_base / freq_factor, freq_scale, corr_dims, i0,
290
+ ext_factor, attn_factor, cos_theta, sin_theta);
291
+
292
+ const float x0 = x[ix + 0];
293
+ const float x1 = x[ix + n_dims];
294
+
295
+ dst[idst + 0] = x0 * cos_theta - x1 * sin_theta;
296
+ dst[idst + n_dims] = x0 * sin_theta + x1 * cos_theta;
297
+ }
298
+
299
+ template <bool forward, typename T, typename D>
300
+ static void
301
+ rope_norm_sycl(const T *x, D *dst, const int ne00, const int ne01,
302
+ const int ne02, const int s01, const int s02, const int s03,
303
+ const int s1, const int s2, const int s3, const int n_dims,
304
+ const int n_offs, const int nr, const int32_t *pos, const float freq_scale,
305
+ const float freq_base, const float ext_factor,
306
+ const float attn_factor, const rope_corr_dims corr_dims,
307
+ const float *freq_factors, const int64_t *row_indices,
308
+ const int set_rows_stride, dpct::queue_ptr stream) {
309
+ GGML_ASSERT(ne00 % 2 == 0);
310
+ const dpct::dim3 block_dims(1, SYCL_ROPE_BLOCK_SIZE, 1);
311
+ const int n_blocks_x =
312
+ (ne00 + 2 * SYCL_ROPE_BLOCK_SIZE - 1) / (2 * SYCL_ROPE_BLOCK_SIZE);
313
+ const dpct::dim3 block_nums(nr, n_blocks_x, 1);
314
+
315
+ const float theta_scale = powf(freq_base, -2.0f / n_dims);
316
+
317
+ if (freq_factors == nullptr) {
318
+ stream->parallel_for(
319
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
320
+ [=](sycl::nd_item<3> item_ct1) {
321
+ GGML_UNUSED(item_ct1);
322
+ rope_norm<forward, false>(
323
+ x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims,
324
+ n_offs, pos, freq_scale, ext_factor, attn_factor, corr_dims,
325
+ theta_scale, freq_factors, row_indices, set_rows_stride);
326
+ });
327
+ } else {
328
+ stream->parallel_for(
329
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
330
+ [=](sycl::nd_item<3> item_ct1) {
331
+ GGML_UNUSED(item_ct1);
332
+ rope_norm<forward, true>(
333
+ x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims,
334
+ n_offs, pos, freq_scale, ext_factor, attn_factor, corr_dims,
335
+ theta_scale, freq_factors, row_indices, set_rows_stride);
336
+ });
337
+ }
338
+ }
339
+
340
+ template <bool forward, typename T, typename D>
341
+ static void
342
+ rope_neox_sycl(const T *x, D *dst, const int ne00, const int ne01,
343
+ const int ne02, const int s01, const int s02, const int s03,
344
+ const int s1, const int s2, const int s3, const int n_dims,
345
+ const int n_offs, const int nr, const int32_t *pos, const float freq_scale,
346
+ const float freq_base, const float ext_factor,
347
+ const float attn_factor, const rope_corr_dims corr_dims,
348
+ const float *freq_factors, const int64_t *row_indices,
349
+ const int set_rows_stride, dpct::queue_ptr stream) {
350
+ GGML_ASSERT(ne00 % 2 == 0);
351
+ const dpct::dim3 block_dims(1, SYCL_ROPE_BLOCK_SIZE, 1);
352
+ const int n_blocks_x =
353
+ (ne00 + 2 * SYCL_ROPE_BLOCK_SIZE - 1) / (2 * SYCL_ROPE_BLOCK_SIZE);
354
+ const dpct::dim3 block_nums(nr, n_blocks_x, 1);
355
+
356
+ const float theta_scale = powf(freq_base, -2.0f / n_dims);
357
+
358
+ if (freq_factors == nullptr) {
359
+ stream->parallel_for(
360
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
361
+ [=](sycl::nd_item<3> item_ct1) {
362
+ GGML_UNUSED(item_ct1);
363
+ rope_neox<forward, false>(
364
+ x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims,
365
+ n_offs, pos, freq_scale, ext_factor, attn_factor, corr_dims,
366
+ theta_scale, freq_factors, row_indices, set_rows_stride);
367
+ });
368
+ } else {
369
+ stream->parallel_for(
370
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
371
+ [=](sycl::nd_item<3> item_ct1) {
372
+ GGML_UNUSED(item_ct1);
373
+ rope_neox<forward, true>(
374
+ x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims,
375
+ n_offs, pos, freq_scale, ext_factor, attn_factor, corr_dims,
376
+ theta_scale, freq_factors, row_indices, set_rows_stride);
377
+ });
378
+ }
379
+ }
380
+
381
+ template <bool forward, typename T>
382
+ static void
383
+ rope_multi_sycl(const T *x, T *dst, const int ne00, const int ne01,
384
+ const int ne02, const int s01, const int s02, const int s03,
385
+ const int s1, const int s2, const int s3, const int n_dims,
386
+ const int n_offs, const int nr, const int32_t *pos, const float freq_scale,
387
+ const float freq_base, const float ext_factor,
388
+ const float attn_factor, const rope_corr_dims corr_dims,
389
+ const float *freq_factors, const mrope_sections sections,
390
+ const bool is_imrope, dpct::queue_ptr stream) {
391
+ GGML_ASSERT(ne00 % 2 == 0);
392
+ const dpct::dim3 block_dims(1, SYCL_ROPE_BLOCK_SIZE, 1);
393
+ const int n_blocks_x =
394
+ (ne00 + 2 * SYCL_ROPE_BLOCK_SIZE - 1) / (2 * SYCL_ROPE_BLOCK_SIZE);
395
+ const dpct::dim3 block_nums(nr, n_blocks_x, 1);
396
+
397
+ const float theta_scale = powf(freq_base, -2.0f / n_dims);
398
+
399
+ if (freq_factors == nullptr) {
400
+ stream->parallel_for(
401
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
402
+ [=](sycl::nd_item<3> item_ct1) {
403
+ GGML_UNUSED(item_ct1);
404
+ rope_multi<forward, false, T>(
405
+ x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims,
406
+ n_offs, pos, freq_scale, ext_factor, attn_factor, corr_dims,
407
+ theta_scale, freq_factors, sections, is_imrope);
408
+ });
409
+ } else {
410
+ stream->parallel_for(
411
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
412
+ [=](sycl::nd_item<3> item_ct1) {
413
+ GGML_UNUSED(item_ct1);
414
+ rope_multi<forward, true, T>(
415
+ x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims,
416
+ n_offs, pos, freq_scale, ext_factor, attn_factor, corr_dims,
417
+ theta_scale, freq_factors, sections, is_imrope);
418
+ });
419
+ }
420
+ }
421
+
422
+ template <bool forward, typename T>
423
+ static void
424
+ rope_vision_sycl(const T *x, T *dst, const int ne00, const int ne01,
425
+ const int ne02, const int s01, const int s02, const int s03,
426
+ const int s1, const int s2, const int s3, const int n_dims,
427
+ const int nr, const int32_t *pos, const float freq_scale,
428
+ const float freq_base, const float ext_factor,
429
+ const float attn_factor, const rope_corr_dims corr_dims,
430
+ const float *freq_factors, const mrope_sections sections,
431
+ dpct::queue_ptr stream) {
432
+ GGML_ASSERT(ne00 % 2 == 0);
433
+ const dpct::dim3 block_dims(1, SYCL_ROPE_BLOCK_SIZE, 1);
434
+ const int n_blocks_x =
435
+ (ne00 + 2 * SYCL_ROPE_BLOCK_SIZE - 1) / (2 * SYCL_ROPE_BLOCK_SIZE);
436
+ const dpct::dim3 block_nums(nr, n_blocks_x, 1);
437
+
438
+ const float theta_scale = powf(freq_base, -2.0f / n_dims);
439
+
440
+ if (freq_factors == nullptr) {
441
+ stream->parallel_for(
442
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
443
+ [=](sycl::nd_item<3> item_ct1) {
444
+ GGML_UNUSED(item_ct1);
445
+ rope_vision<forward, false, T>(
446
+ x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims,
447
+ pos, freq_scale, ext_factor, attn_factor, corr_dims,
448
+ theta_scale, freq_factors, sections);
449
+ });
450
+ } else {
451
+ stream->parallel_for(
452
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
453
+ [=](sycl::nd_item<3> item_ct1) {
454
+ GGML_UNUSED(item_ct1);
455
+ rope_vision<forward, true, T>(
456
+ x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims,
457
+ pos, freq_scale, ext_factor, attn_factor, corr_dims,
458
+ theta_scale, freq_factors, sections);
459
+ });
460
+ }
461
+ }
462
+
463
+ template <bool forward>
464
+ void ggml_sycl_op_rope_impl(ggml_backend_sycl_context &ctx, ggml_tensor *dst,
465
+ const ggml_tensor *set_rows = nullptr) {
466
+ const ggml_tensor *src0 = dst->src[0];
467
+ const ggml_tensor *src1 = dst->src[1];
468
+ const ggml_tensor *src2 = dst->src[2];
469
+
470
+ const float *src0_d = (const float *)src0->data;
471
+ const float *src1_d = (const float *)src1->data;
472
+
473
+ void *dst_d = dst->data;
474
+ const int64_t *row_indices = nullptr;
475
+ ggml_type dst_type = dst->type;
476
+ int set_rows_stride = 0;
477
+
478
+ if (set_rows != nullptr) {
479
+ GGML_ASSERT(forward);
480
+ dst_d = set_rows->data;
481
+ row_indices = (const int64_t *)set_rows->src[1]->data;
482
+ dst_type = set_rows->type;
483
+ set_rows_stride = set_rows->nb[1] / ggml_type_size(set_rows->type);
484
+ }
485
+ dpct::queue_ptr stream = ctx.stream();
486
+
487
+ GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);
488
+ GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
489
+ GGML_ASSERT(src0->type == dst->type ||
490
+ (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16));
491
+
492
+ const int64_t ne00 = src0->ne[0]; // head dims
493
+ const int64_t ne01 = src0->ne[1]; // num heads
494
+ const int64_t ne02 = src0->ne[2]; // num heads
495
+ const int64_t nr = ggml_nrows(src0);
496
+
497
+ const size_t s01 = src0->nb[1] / ggml_type_size(src0->type);
498
+ const size_t s02 = src0->nb[2] / ggml_type_size(src0->type);
499
+ const size_t s03 = src0->nb[3] / ggml_type_size(src0->type);
500
+
501
+ const size_t s1 = dst->nb[1] / ggml_type_size(dst->type);
502
+ const size_t s2 = dst->nb[2] / ggml_type_size(dst->type);
503
+ const size_t s3 = dst->nb[3] / ggml_type_size(dst->type);
504
+
505
+ const int n_dims = ((int32_t *)dst->op_params)[1];
506
+ const int mode = ((int32_t *)dst->op_params)[2];
507
+ const int n_ctx_orig = ((int32_t *)dst->op_params)[4];
508
+ const int n_offs = ((int32_t *)dst->op_params)[15];
509
+ mrope_sections sections;
510
+
511
+ float freq_base;
512
+ float freq_scale;
513
+ float ext_factor;
514
+ float attn_factor;
515
+ float beta_fast;
516
+ float beta_slow;
517
+
518
+ memcpy(&freq_base, (int32_t *)dst->op_params + 5, sizeof(float));
519
+ memcpy(&freq_scale, (int32_t *)dst->op_params + 6, sizeof(float));
520
+ memcpy(&ext_factor, (int32_t *)dst->op_params + 7, sizeof(float));
521
+ memcpy(&attn_factor, (int32_t *)dst->op_params + 8, sizeof(float));
522
+ memcpy(&beta_fast, (int32_t *)dst->op_params + 9, sizeof(float));
523
+ memcpy(&beta_slow, (int32_t *)dst->op_params + 10, sizeof(float));
524
+ memcpy(&sections.v, (int32_t *)dst->op_params + 11, sizeof(int) * 4);
525
+
526
+ const bool is_neox = mode & GGML_ROPE_TYPE_NEOX;
527
+ const bool is_mrope = mode & GGML_ROPE_TYPE_MROPE;
528
+ const bool is_imrope = mode == GGML_ROPE_TYPE_IMROPE;
529
+ const bool is_vision = mode == GGML_ROPE_TYPE_VISION;
530
+
531
+ if (is_mrope) {
532
+ GGML_ASSERT(sections.v[0] > 0 || sections.v[1] > 0 ||
533
+ sections.v[2] > 0);
534
+ }
535
+
536
+ if (is_vision) {
537
+ GGML_ASSERT(n_dims == ne00 / 2);
538
+ GGML_ASSERT(n_offs == 0); // offset not supported for vision, as the rotated pairs span the whole row
539
+ }
540
+
541
+ const int32_t *pos = (const int32_t *)src1_d;
542
+
543
+ const float *freq_factors = nullptr;
544
+ if (src2 != nullptr) {
545
+ freq_factors = (const float *)src2->data;
546
+ }
547
+
548
+ rope_corr_dims corr_dims;
549
+ ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast,
550
+ beta_slow, corr_dims.v);
551
+
552
+ // compute
553
+ if (is_neox) {
554
+ GGML_SYCL_DEBUG("%s: neox path\n", __func__);
555
+ if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F32) {
556
+ rope_neox_sycl<forward, float, float>(
557
+ (const float *)src0_d, (float *)dst_d, ne00, ne01, ne02, s01,
558
+ s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
559
+ ext_factor, attn_factor, corr_dims, freq_factors, row_indices,
560
+ set_rows_stride, stream);
561
+ } else if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F16) {
562
+ rope_neox_sycl<forward, float, sycl::half>(
563
+ (const float *)src0_d, (sycl::half *)dst_d, ne00, ne01, ne02,
564
+ s01, s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale,
565
+ freq_base, ext_factor, attn_factor, corr_dims, freq_factors,
566
+ row_indices, set_rows_stride, stream);
567
+ } else if (src0->type == GGML_TYPE_F16 && dst_type == GGML_TYPE_F16) {
568
+ rope_neox_sycl<forward, sycl::half, sycl::half>(
569
+ (const sycl::half *)src0_d, (sycl::half *)dst_d, ne00, ne01,
570
+ ne02, s01, s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale,
571
+ freq_base, ext_factor, attn_factor, corr_dims, freq_factors,
572
+ row_indices, set_rows_stride, stream);
573
+ } else {
574
+ GGML_ABORT("Fatal error: Tensor type unsupported!");
575
+ }
576
+ } else if (is_mrope && !is_vision) {
577
+ GGML_SYCL_DEBUG("%s: mrope path\n", __func__);
578
+ if (src0->type == GGML_TYPE_F32) {
579
+ rope_multi_sycl<forward>((const float *)src0_d, (float *)dst_d,
580
+ ne00, ne01, ne02, s01, s02, s03, s1, s2,
581
+ s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
582
+ ext_factor, attn_factor, corr_dims,
583
+ freq_factors, sections, is_imrope, stream);
584
+ } else if (src0->type == GGML_TYPE_F16) {
585
+ rope_multi_sycl<forward>(
586
+ (const sycl::half *)src0_d, (sycl::half *)dst_d, ne00, ne01,
587
+ ne02, s01, s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale,
588
+ freq_base, ext_factor, attn_factor, corr_dims, freq_factors,
589
+ sections, is_imrope, stream);
590
+ } else {
591
+ GGML_ABORT("Fatal error: Tensor type unsupported!");
592
+ }
593
+ } else if (is_vision) {
594
+ GGML_SYCL_DEBUG("%s: vision path\n", __func__);
595
+ if (src0->type == GGML_TYPE_F32) {
596
+ rope_vision_sycl<forward>(
597
+ (const float *)src0_d, (float *)dst_d, ne00, ne01, ne02, s01,
598
+ s02, s03, s1, s2, s3, n_dims, nr, pos, freq_scale, freq_base,
599
+ ext_factor, attn_factor, corr_dims, freq_factors, sections,
600
+ stream);
601
+ } else if (src0->type == GGML_TYPE_F16) {
602
+ rope_vision_sycl<forward>(
603
+ (const sycl::half *)src0_d, (sycl::half *)dst_d, ne00, ne01,
604
+ ne02, s01, s02, s03, s1, s2, s3, n_dims, nr, pos, freq_scale,
605
+ freq_base, ext_factor, attn_factor, corr_dims, freq_factors,
606
+ sections, stream);
607
+ } else {
608
+ GGML_ABORT("Fatal error: Tensor type unsupported!");
609
+ }
610
+ } else {
611
+ GGML_SYCL_DEBUG("%s: norm path\n", __func__);
612
+ if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F32) {
613
+ rope_norm_sycl<forward, float, float>(
614
+ (const float *)src0_d, (float *)dst_d, ne00, ne01, ne02, s01,
615
+ s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
616
+ ext_factor, attn_factor, corr_dims, freq_factors, row_indices,
617
+ set_rows_stride, stream);
618
+ } else if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F16) {
619
+ rope_norm_sycl<forward, float, sycl::half>(
620
+ (const float *)src0_d, (sycl::half *)dst_d, ne00, ne01, ne02,
621
+ s01, s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale,
622
+ freq_base, ext_factor, attn_factor, corr_dims, freq_factors,
623
+ row_indices, set_rows_stride, stream);
624
+ } else if (src0->type == GGML_TYPE_F16 && dst_type == GGML_TYPE_F16) {
625
+ rope_norm_sycl<forward, sycl::half, sycl::half>(
626
+ (const sycl::half *)src0_d, (sycl::half *)dst_d, ne00, ne01,
627
+ ne02, s01, s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale,
628
+ freq_base, ext_factor, attn_factor, corr_dims, freq_factors,
629
+ row_indices, set_rows_stride, stream);
630
+ } else {
631
+ GGML_ABORT("Fatal error: Tensor type unsupported!");
632
+ }
633
+ }
634
+ }
635
+
636
+ void ggml_sycl_rope(ggml_backend_sycl_context &ctx, ggml_tensor *dst) {
637
+ scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/3);
638
+
639
+ ggml_sycl_op_rope_impl<true>(ctx, dst);
640
+ }
641
+
642
+ void ggml_sycl_rope_back(ggml_backend_sycl_context &ctx, ggml_tensor *dst) {
643
+ scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/3);
644
+ ggml_sycl_op_rope_impl<false>(ctx, dst);
645
+ }
646
+
647
+ void ggml_sycl_rope_fused(ggml_backend_sycl_context &ctx, ggml_tensor *rope,
648
+ ggml_tensor *set_rows) {
649
+ scope_op_debug_print scope_dbg_print(__func__, rope, /*num_src=*/3);
650
+ ggml_sycl_op_rope_impl<true>(ctx, rope, set_rows);
651
+ }
ggml/src/ggml-sycl/rope.hpp ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //
2
+ // MIT license
3
+ // Copyright (C) 2024 Intel Corporation
4
+ // SPDX-License-Identifier: MIT
5
+ //
6
+
7
+ //
8
+ // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
9
+ // See https://llvm.org/LICENSE.txt for license information.
10
+ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
11
+ //
12
+
13
+ #ifndef GGML_SYCL_ROPE_HPP
14
+ #define GGML_SYCL_ROPE_HPP
15
+
16
+ #include "common.hpp"
17
+
18
+ #define SYCL_ROPE_BLOCK_SIZE 256
19
+
20
+ void ggml_sycl_rope(ggml_backend_sycl_context & ctx, ggml_tensor *dst);
21
+
22
+ void ggml_sycl_rope_back(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
23
+
24
+ void ggml_sycl_rope_fused(ggml_backend_sycl_context & ctx, ggml_tensor * dst, ggml_tensor * set_rows);
25
+
26
+ #endif // GGML_SYCL_ROPE_HPP
ggml/src/ggml-sycl/set.cpp ADDED
@@ -0,0 +1,73 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #include "presets.hpp"
2
+ #include "common.hpp"
3
+ #include "ggml.h"
4
+ #include "set.hpp"
5
+ #include <cstdint>
6
+ #include <sycl/sycl.hpp>
7
+ using namespace sycl;
8
+
9
+ // Internal function: perform element-wise set operation for each thread
10
+ inline void set_f32(const float* src, float* dst,
11
+ const int64_t ne0, const int64_t ne1,
12
+ const int64_t ne2, const int64_t ne3,
13
+ const int64_t nb[3], const int64_t src_nb[3],
14
+ const int64_t offset_elem,
15
+ const nd_item<1>& item)
16
+ {
17
+ const size_t idx = item.get_global_id(0);
18
+ const size_t total = ne0 * ne1 * ne2 * ne3;
19
+ if (idx >= total) return;
20
+
21
+ // Convert linear index to 4D indices
22
+ const size_t i3 = idx / (ne2 * ne1 * ne0);
23
+ const size_t rem = idx % (ne2 * ne1 * ne0);
24
+ const size_t i2 = rem / (ne1 * ne0);
25
+ const size_t rem2 = rem % (ne1 * ne0);
26
+ const size_t i1 = rem2 / ne0;
27
+ const size_t i0 = rem2 % ne0;
28
+
29
+ // Compute source and destination indices and copy
30
+ dst[i0 + i1*nb[0] + i2*nb[1] + i3*nb[2] + offset_elem] =
31
+ src[i0 + i1*src_nb[0] + i2*src_nb[1] + i3*src_nb[2]];
32
+ }
33
+
34
+ // Main function: prepare GPU queue and launch parallel_for
35
+ void ggml_sycl_op_set(ggml_backend_sycl_context& ctx, ggml_tensor* dst) {
36
+ const ggml_tensor* src0 = dst->src[0];
37
+ const ggml_tensor* src1 = dst->src[1];
38
+
39
+ // Ensure shapes and types are compatible
40
+ GGML_ASSERT(ggml_are_same_shape(src0, dst));
41
+ GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0));
42
+ GGML_ASSERT(dst->type == src0->type && src0->type == src1->type && dst->type == GGML_TYPE_F32);
43
+
44
+ const int32_t* opts = (const int32_t*) dst->op_params;
45
+ const int64_t nb[3] = {opts[0]/sizeof(float), opts[1]/sizeof(float), opts[2]/sizeof(float)};
46
+ const int64_t offset_elem = opts[3] / sizeof(float);
47
+ const bool inplace = opts[4];
48
+
49
+ float* dst_ptr = (float*) dst->data;
50
+ const float* src0_ptr = (const float*) src0->data;
51
+ const float* src1_ptr = (const float*) src1->data;
52
+
53
+ queue_ptr stream = ctx.stream();
54
+
55
+ // Copy src0 to dst if not inplace
56
+ if (!inplace)
57
+ stream->memcpy(dst_ptr, src0_ptr, ggml_nbytes(dst));
58
+
59
+ const int64_t ne[4] = {src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3]};
60
+ const int64_t src_nb[3] = {src1->nb[1]/sizeof(float), src1->nb[2]/sizeof(float), src1->nb[3]/sizeof(float)};
61
+
62
+ const size_t total_threads = ne[0]*ne[1]*ne[2]*ne[3];
63
+ const size_t grid_size = ((total_threads + SYCL_SET_BLOCK_SIZE - 1) / SYCL_SET_BLOCK_SIZE) * SYCL_SET_BLOCK_SIZE;
64
+
65
+ // Copy src0 to dst if not inplace
66
+ stream->parallel_for(
67
+ nd_range<1>(range<1>(grid_size), range<1>(SYCL_SET_BLOCK_SIZE)),
68
+ [=](nd_item<1> item) {
69
+ set_f32(src1_ptr, dst_ptr,
70
+ ne[0], ne[1], ne[2], ne[3],
71
+ nb, src_nb, offset_elem, item); }
72
+ );
73
+ }
ggml/src/ggml-sycl/set.hpp ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ #pragma once
2
+ #include "backend.hpp"
3
+ #include "ggml.h"
4
+
5
+ void ggml_sycl_op_set(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
ggml/src/ggml-sycl/set_rows.cpp ADDED
@@ -0,0 +1,577 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #include "set_rows.hpp"
2
+ #include "cpy.hpp"
3
+
4
+ #include "ggml-quants.h"
5
+
6
+ #include <vector>
7
+
8
+ namespace utils {
9
+ template<typename T>
10
+ static constexpr bool is_arithmetic_v() {
11
+ return std::is_arithmetic_v<T> || std::is_same_v<T, sycl::half>
12
+ #ifdef GGML_SYCL_HAS_BF16
13
+ || std::is_same_v<T, sycl::ext::oneapi::bfloat16>
14
+ #endif
15
+ ;
16
+ }
17
+ }
18
+
19
+ template<typename TIn, typename TOut>
20
+ static inline std::enable_if_t<utils::is_arithmetic_v<TIn>() && utils::is_arithmetic_v<TOut>(), void>
21
+ convert (const char* src, char* dst) {
22
+ auto src_val = *reinterpret_cast<const TIn*>(src);
23
+ auto dst_val = sycl::vec<TIn, 1>(src_val).template convert<TOut, sycl::rounding_mode::automatic>()[0];
24
+ *reinterpret_cast<TOut*>(dst) = dst_val;
25
+ }
26
+
27
+ #ifdef GGML_SYCL_HAS_BF16
28
+ // sycl::vec::convert does not provide a half -> bfloat16 path, so route through float.
29
+ template<>
30
+ inline void convert<sycl::half, sycl::ext::oneapi::bfloat16>(const char* src, char* dst) {
31
+ const float tmp = sycl::vec<sycl::half, 1>(*reinterpret_cast<const sycl::half*>(src))
32
+ .template convert<float, sycl::rounding_mode::automatic>()[0];
33
+ *reinterpret_cast<sycl::ext::oneapi::bfloat16*>(dst) = sycl::ext::oneapi::bfloat16(tmp);
34
+ }
35
+ #endif
36
+
37
+ template <typename TIn, typename TIdx, typename blockType, int qk, cpy_kernel_t cpyblck>
38
+ static void set_rows_sycl_q(const char * __restrict__ src0_d,
39
+ const TIdx * __restrict__ src1_d,
40
+ blockType * __restrict__ dst_d,
41
+ // tensor dimensions src0 and src1
42
+ const int64_t ne00,
43
+ const int64_t ne01,
44
+ const int64_t ne02,
45
+ const int64_t ne03,
46
+ const int64_t ne10,
47
+ const int64_t ne11,
48
+ const int64_t ne12,
49
+ const int64_t ne13,
50
+ // strides for src0
51
+ const size_t nb00,
52
+ const size_t nb01,
53
+ const size_t nb02,
54
+ const size_t nb03,
55
+ // strides for src1
56
+ const size_t nb10,
57
+ const size_t nb11,
58
+ const size_t nb12,
59
+ const size_t nb13,
60
+ // strides for dst
61
+ const size_t nb1,
62
+ const size_t nb2,
63
+ const size_t nb3,
64
+ queue_ptr stream) {
65
+ const int64_t total_blocks = (ne00 * ne01 * ne02 * ne03) / qk;
66
+ constexpr int block_size = 256;
67
+ const int64_t grid_size = ceil_div(total_blocks, block_size);
68
+
69
+ stream->parallel_for(sycl::nd_range<1>(grid_size * block_size, block_size), [=](sycl::nd_item<1> item_ct1) {
70
+ const int64_t i = item_ct1.get_global_linear_id();
71
+ if (i >= total_blocks) {
72
+ return;
73
+ }
74
+ const int64_t i_base = i * qk;
75
+ const int64_t i03 = i_base / (ne00 * ne01 * ne02);
76
+ const int64_t rem1 = i_base - i03 * (ne00 * ne01 * ne02);
77
+ const int64_t i02 = rem1 / (ne00 * ne01);
78
+ const int64_t rem2 = rem1 - i02 * ne00 * ne01;
79
+ const int64_t i01 = rem2 / ne00;
80
+ const int64_t i00 = rem2 - i01 * ne00;
81
+ const int64_t i12 = i03 % ne12;
82
+ const int64_t i11 = i02 % ne11;
83
+ const int64_t i10 = i01;
84
+ const size_t src_offset = calculate_offset<3>({ nb01, nb02, nb03 }, { i01, i02, i03 });
85
+ const char * src_block = src0_d + src_offset + i00 * sizeof(TIn);
86
+ const size_t src1_offset = calculate_offset<3>({ nb10, nb11, nb12 }, { i10, i11, i12 });
87
+ const int64_t dst_row = src1_d[src1_offset / sizeof(TIdx)];
88
+ const size_t dst_offset =
89
+ calculate_offset<3>({ nb1, nb2, nb3 }, { dst_row, i02, i03 }) + (i00 / qk) * sizeof(blockType);
90
+ char * dst_block = reinterpret_cast<char *>(reinterpret_cast<char *>(dst_d) + dst_offset);
91
+ if constexpr (std::is_same_v<TIn, float>) {
92
+ cpyblck(src_block, dst_block);
93
+ } else {
94
+ float src_block_f32[qk];
95
+ const TIn * src_block_t = reinterpret_cast<const TIn *>(src_block);
96
+ for (int j = 0; j < qk; ++j) {
97
+ src_block_f32[j] = (float) src_block_t[j];
98
+ }
99
+ cpyblck(reinterpret_cast<const char *>(src_block_f32), dst_block);
100
+ }
101
+ });
102
+ GGML_UNUSED(ne10);
103
+ GGML_UNUSED(ne13);
104
+ GGML_UNUSED(nb00);
105
+ GGML_UNUSED(nb13);
106
+ }
107
+
108
+ template<typename blockType>
109
+ using quantize_row_qk_t = void (*)(const float *, blockType *, int64_t);
110
+
111
+ using quantize_rows_f_t = size_t (*)(const float *, void *, int64_t, int64_t, const float *);
112
+
113
+ template <typename TIn, typename TIdx, typename blockType, int qk, quantize_row_qk_t<blockType> quantize_row>
114
+ static void set_rows_sycl_qk_host(
115
+ const ggml_tensor * src0,
116
+ const ggml_tensor * src1,
117
+ ggml_tensor * dst,
118
+ const int64_t ne00,
119
+ const int64_t ne01,
120
+ const int64_t ne02,
121
+ const int64_t ne03,
122
+ const int64_t ne11,
123
+ const int64_t ne12,
124
+ const size_t nb01,
125
+ const size_t nb02,
126
+ const size_t nb03,
127
+ const size_t nb10,
128
+ const size_t nb11,
129
+ const size_t nb12,
130
+ const size_t nb1,
131
+ const size_t nb2,
132
+ const size_t nb3,
133
+ queue_ptr stream) {
134
+ GGML_ASSERT(ne00 % qk == 0);
135
+
136
+ const size_t src0_bytes = ggml_nbytes(src0);
137
+ const size_t src1_bytes = ggml_nbytes(src1);
138
+
139
+ std::vector<char> src0_host(src0_bytes);
140
+ std::vector<char> src1_host(src1_bytes);
141
+
142
+ stream->memcpy(src0_host.data(), src0->data, src0_bytes);
143
+ stream->memcpy(src1_host.data(), src1->data, src1_bytes);
144
+ stream->wait();
145
+
146
+ std::vector<float> src_row_f32(ne00);
147
+ const int64_t nblocks = ne00 / qk;
148
+ std::vector<blockType> dst_row_q(nblocks);
149
+
150
+ for (int64_t i03 = 0; i03 < ne03; ++i03) {
151
+ for (int64_t i02 = 0; i02 < ne02; ++i02) {
152
+ for (int64_t i01 = 0; i01 < ne01; ++i01) {
153
+ const int64_t i12 = i03 % ne12;
154
+ const int64_t i11 = i02 % ne11;
155
+ const int64_t i10 = i01;
156
+
157
+ const size_t src1_offset = calculate_offset<3>({ nb10, nb11, nb12 }, { i10, i11, i12 });
158
+ const int64_t dst_row = *(const TIdx *) (src1_host.data() + src1_offset);
159
+
160
+ const size_t src0_row_offset = calculate_offset<3>({ nb01, nb02, nb03 }, { i01, i02, i03 });
161
+ const TIn * src_row = reinterpret_cast<const TIn *>(src0_host.data() + src0_row_offset);
162
+
163
+ for (int64_t i00 = 0; i00 < ne00; ++i00) {
164
+ src_row_f32[i00] = (float) src_row[i00];
165
+ }
166
+
167
+ quantize_row(src_row_f32.data(), dst_row_q.data(), ne00);
168
+
169
+ const size_t dst_offset = calculate_offset<3>({ nb1, nb2, nb3 }, { dst_row, i02, i03 });
170
+ stream->memcpy((char *) dst->data + dst_offset, dst_row_q.data(), nblocks * sizeof(blockType));
171
+ stream->wait();
172
+ }
173
+ }
174
+ }
175
+ }
176
+
177
+ template <typename TIn, typename TIdx, typename blockType, int qk, quantize_rows_f_t quantize_rows>
178
+ static void set_rows_sycl_iq_host(
179
+ const ggml_tensor * src0,
180
+ const ggml_tensor * src1,
181
+ ggml_tensor * dst,
182
+ const int64_t ne00,
183
+ const int64_t ne01,
184
+ const int64_t ne02,
185
+ const int64_t ne03,
186
+ const int64_t ne11,
187
+ const int64_t ne12,
188
+ const size_t nb01,
189
+ const size_t nb02,
190
+ const size_t nb03,
191
+ const size_t nb10,
192
+ const size_t nb11,
193
+ const size_t nb12,
194
+ const size_t nb1,
195
+ const size_t nb2,
196
+ const size_t nb3,
197
+ queue_ptr stream) {
198
+ GGML_ASSERT(ne00 % qk == 0);
199
+
200
+ const size_t src0_bytes = ggml_nbytes(src0);
201
+ const size_t src1_bytes = ggml_nbytes(src1);
202
+
203
+ std::vector<char> src0_host(src0_bytes);
204
+ std::vector<char> src1_host(src1_bytes);
205
+
206
+ stream->memcpy(src0_host.data(), src0->data, src0_bytes);
207
+ stream->memcpy(src1_host.data(), src1->data, src1_bytes);
208
+ stream->wait();
209
+
210
+ std::vector<float> src_row_f32(ne00);
211
+ const int64_t nblocks = ne00 / qk;
212
+ std::vector<blockType> dst_row_q(nblocks);
213
+
214
+ for (int64_t i03 = 0; i03 < ne03; ++i03) {
215
+ for (int64_t i02 = 0; i02 < ne02; ++i02) {
216
+ for (int64_t i01 = 0; i01 < ne01; ++i01) {
217
+ const int64_t i12 = i03 % ne12;
218
+ const int64_t i11 = i02 % ne11;
219
+ const int64_t i10 = i01;
220
+
221
+ const size_t src1_offset = calculate_offset<3>({ nb10, nb11, nb12 }, { i10, i11, i12 });
222
+ const int64_t dst_row = *(const TIdx *) (src1_host.data() + src1_offset);
223
+
224
+ const size_t src0_row_offset = calculate_offset<3>({ nb01, nb02, nb03 }, { i01, i02, i03 });
225
+ const TIn * src_row = reinterpret_cast<const TIn *>(src0_host.data() + src0_row_offset);
226
+
227
+ for (int64_t i00 = 0; i00 < ne00; ++i00) {
228
+ src_row_f32[i00] = (float) src_row[i00];
229
+ }
230
+
231
+ quantize_rows(src_row_f32.data(), dst_row_q.data(), 1, ne00, nullptr);
232
+
233
+ const size_t dst_offset = calculate_offset<3>({ nb1, nb2, nb3 }, { dst_row, i02, i03 });
234
+ stream->memcpy((char *) dst->data + dst_offset, dst_row_q.data(), nblocks * sizeof(blockType));
235
+ stream->wait();
236
+ }
237
+ }
238
+ }
239
+ }
240
+
241
+ template<typename TIn, typename TIdx, typename TOut>
242
+ static void k_set_rows(
243
+ const char * __restrict__ src0, const TIdx * __restrict__ src1, char * __restrict__ dst,
244
+ const int64_t ne00, const int64_t ne01, const int64_t ne02,
245
+ const int64_t ne11, const int64_t ne12,
246
+ const size_t nb01, const size_t nb02, const size_t nb03,
247
+ const size_t nb10, const size_t nb11, const size_t nb12,
248
+ const size_t nb1, const size_t nb2, const size_t nb3,
249
+ const size_t src_type_size, const size_t dst_type_size,
250
+ const int64_t total_elements,
251
+ const sycl::nd_item<1> & item_ct1) {
252
+
253
+ const int64_t i = item_ct1.get_global_linear_id();
254
+ if (i >= total_elements) {
255
+ return;
256
+ }
257
+
258
+ const int64_t i03 = i / (ne00 * ne01 * ne02);
259
+ const int64_t i02 = (i - i03 * ne00 * ne01 * ne02) / (ne00 * ne01);
260
+ const int64_t i01 = (i - i03 * ne00 * ne01 * ne02 - i02 * ne00 * ne01) / ne00;
261
+ const int64_t i00 = i - i03 * ne00 * ne01 * ne02 - i02 * ne00 * ne01 - i01 * ne00;
262
+
263
+ const int64_t i12 = i03 % ne12;
264
+ const int64_t i11 = i02 % ne11;
265
+ const int64_t i10 = i01;
266
+
267
+ const int64_t dst_row = *(const TIdx *)((const char *)src1 + calculate_offset<3>({nb10, nb11, nb12}, {i10, i11, i12}));
268
+
269
+ const char * src0_row = src0 + calculate_offset<3>({nb01, nb02, nb03}, {i01, i02, i03});
270
+ const char * src_elem = src0_row + i00 * src_type_size;
271
+ char * dst_row_ptr = dst + dst_row*nb1 + i02*nb2 + i03*nb3;
272
+ char * dst_elem = dst_row_ptr + i00 * dst_type_size;
273
+
274
+ convert<TIn, TOut>(src_elem, dst_elem);
275
+ }
276
+
277
+ template<typename TIn, typename TIdx, typename TOut>
278
+ static void set_rows_sycl(
279
+ const char * src0_d, const TIdx * src1_d, char * dst_d,
280
+ const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03,
281
+ const int64_t ne11, const int64_t ne12, const size_t nb01, const size_t nb02, const size_t nb03,
282
+ const size_t nb10, const size_t nb11, const size_t nb12,
283
+ const size_t nb1, const size_t nb2, const size_t nb3,
284
+ const size_t src_type_size, const size_t dst_type_size,
285
+ queue_ptr stream) {
286
+
287
+ const int64_t total_elements = ne00 * ne01 * ne02 * ne03;
288
+
289
+ constexpr int block_size = 64;
290
+ const int64_t grid_size = ceil_div(total_elements, block_size);
291
+
292
+ stream->parallel_for(
293
+ sycl::nd_range<1>(grid_size * block_size, block_size),
294
+ [=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
295
+ k_set_rows<TIn, TIdx, TOut>(
296
+ src0_d, src1_d, dst_d,
297
+ ne00, ne01, ne02,
298
+ ne11, ne12,
299
+ nb01, nb02, nb03,
300
+ nb10, nb11, nb12,
301
+ nb1, nb2, nb3,
302
+ src_type_size, dst_type_size,
303
+ total_elements,
304
+ item_ct1
305
+ );
306
+ }
307
+ );
308
+ }
309
+
310
+ template<typename TIn, typename TIdx>
311
+ static void set_rows_sycl(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
312
+ const char * src0_d = (const char *)src0->data;
313
+ const TIdx * src1_d = (const TIdx *)src1->data;
314
+
315
+ GGML_TENSOR_BINARY_OP_LOCALS
316
+
317
+ dpct::queue_ptr stream = ctx.stream();
318
+ switch (dst->type) {
319
+ case GGML_TYPE_F32:
320
+ set_rows_sycl<TIn, TIdx, float>(
321
+ src0_d, src1_d, (char *)dst->data,
322
+ ne00, ne01, ne02, ne03,
323
+ ne11, ne12,
324
+ nb01, nb02, nb03,
325
+ nb10, nb11, nb12,
326
+ nb1, nb2, nb3,
327
+ sizeof(TIn), sizeof(float),
328
+ stream
329
+ );
330
+ break;
331
+ case GGML_TYPE_F16:
332
+ dpct::has_capability_or_fail(stream->get_device(), { sycl::aspect::fp16 });
333
+ set_rows_sycl<TIn, TIdx, sycl::half>(
334
+ src0_d, src1_d, (char *)dst->data,
335
+ ne00, ne01, ne02, ne03,
336
+ ne11, ne12,
337
+ nb01, nb02, nb03,
338
+ nb10, nb11, nb12,
339
+ nb1, nb2, nb3,
340
+ sizeof(TIn), sizeof(sycl::half),
341
+ stream
342
+ );
343
+ break;
344
+ #ifdef GGML_SYCL_HAS_BF16
345
+ case GGML_TYPE_BF16:
346
+ set_rows_sycl<TIn, TIdx, sycl::ext::oneapi::bfloat16>(
347
+ src0_d, src1_d, (char *)dst->data,
348
+ ne00, ne01, ne02, ne03,
349
+ ne11, ne12,
350
+ nb01, nb02, nb03,
351
+ nb10, nb11, nb12,
352
+ nb1, nb2, nb3,
353
+ sizeof(TIn), sizeof(sycl::ext::oneapi::bfloat16),
354
+ stream
355
+ );
356
+ break;
357
+ #endif
358
+ case GGML_TYPE_Q8_0:
359
+ set_rows_sycl_q<TIn, TIdx, block_q8_0, QK8_0, cpy_blck_f32_q8_0>(
360
+ src0_d, src1_d, (block_q8_0 *) dst->data, ne00, ne01, ne02, ne03,
361
+ ne10, ne11, ne12, ne13, nb00, nb01,
362
+ nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
363
+ break;
364
+ case GGML_TYPE_Q1_0:
365
+ set_rows_sycl_q<TIn, TIdx, block_q1_0, QK1_0, cpy_blck_f32_q1_0>(
366
+ src0_d, src1_d, (block_q1_0 *) dst->data, ne00, ne01, ne02, ne03,
367
+ ne10, ne11, ne12, ne13, nb00, nb01,
368
+ nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
369
+ break;
370
+ case GGML_TYPE_Q2_0:
371
+ set_rows_sycl_q<TIn, TIdx, block_q2_0, QK2_0, cpy_blck_f32_q2_0>(
372
+ src0_d, src1_d, (block_q2_0 *) dst->data, ne00, ne01, ne02, ne03,
373
+ ne10, ne11, ne12, ne13, nb00, nb01,
374
+ nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
375
+ break;
376
+ case GGML_TYPE_Q5_1:
377
+ set_rows_sycl_q<TIn, TIdx, block_q5_1, QK5_1, cpy_blck_f32_q5_1>(
378
+ src0_d, src1_d, (block_q5_1 *) dst->data, ne00, ne01, ne02, ne03,
379
+ ne10, ne11, ne12, ne13, nb00, nb01,
380
+ nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
381
+ break;
382
+ case GGML_TYPE_Q5_0:
383
+ set_rows_sycl_q<TIn, TIdx, block_q5_0, QK5_0, cpy_blck_f32_q5_0>(
384
+ src0_d, src1_d, (block_q5_0 *) dst->data, ne00, ne01, ne02, ne03,
385
+ ne10, ne11, ne12, ne13, nb00, nb01,
386
+ nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
387
+ break;
388
+ case GGML_TYPE_Q4_1:
389
+ set_rows_sycl_q<TIn, TIdx, block_q4_1, QK4_1, cpy_blck_f32_q4_1>(
390
+ src0_d, src1_d, (block_q4_1 *) dst->data, ne00, ne01, ne02, ne03,
391
+ ne10, ne11, ne12, ne13, nb00, nb01,
392
+ nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
393
+ break;
394
+ case GGML_TYPE_Q4_0:
395
+ set_rows_sycl_q<TIn, TIdx, block_q4_0, QK4_0, cpy_blck_f32_q4_0>(
396
+ src0_d, src1_d, (block_q4_0 *) dst->data, ne00, ne01, ne02, ne03,
397
+ ne10, ne11, ne12, ne13, nb00, nb01,
398
+ nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
399
+ break;
400
+ case GGML_TYPE_IQ4_NL:
401
+ set_rows_sycl_q<TIn, TIdx, block_iq4_nl, QK4_NL, cpy_blck_f32_iq4_nl>(
402
+ src0_d, src1_d, (block_iq4_nl *) dst->data, ne00, ne01, ne02, ne03,
403
+ ne10, ne11, ne12, ne13, nb00, nb01,
404
+ nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
405
+ break;
406
+ case GGML_TYPE_MXFP4:
407
+ set_rows_sycl_q<TIn, TIdx, block_mxfp4, QK_MXFP4, cpy_blck_f32_mxfp4>(
408
+ src0_d, src1_d, (block_mxfp4 *) dst->data, ne00, ne01, ne02, ne03,
409
+ ne10, ne11, ne12, ne13, nb00, nb01,
410
+ nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
411
+ break;
412
+ case GGML_TYPE_NVFP4:
413
+ set_rows_sycl_q<TIn, TIdx, block_nvfp4, QK_NVFP4, cpy_blck_f32_nvfp4>(
414
+ src0_d, src1_d, (block_nvfp4 *) dst->data, ne00, ne01, ne02, ne03,
415
+ ne10, ne11, ne12, ne13, nb00, nb01,
416
+ nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
417
+ break;
418
+ case GGML_TYPE_Q2_K:
419
+ set_rows_sycl_qk_host<TIn, TIdx, block_q2_K, QK_K, quantize_row_q2_K_ref>(
420
+ src0, src1, dst,
421
+ ne00, ne01, ne02, ne03,
422
+ ne11, ne12,
423
+ nb01, nb02, nb03,
424
+ nb10, nb11, nb12,
425
+ nb1, nb2, nb3,
426
+ stream);
427
+ break;
428
+ case GGML_TYPE_Q3_K:
429
+ set_rows_sycl_qk_host<TIn, TIdx, block_q3_K, QK_K, quantize_row_q3_K_ref>(
430
+ src0, src1, dst,
431
+ ne00, ne01, ne02, ne03,
432
+ ne11, ne12,
433
+ nb01, nb02, nb03,
434
+ nb10, nb11, nb12,
435
+ nb1, nb2, nb3,
436
+ stream);
437
+ break;
438
+ case GGML_TYPE_Q4_K:
439
+ set_rows_sycl_qk_host<TIn, TIdx, block_q4_K, QK_K, quantize_row_q4_K_ref>(
440
+ src0, src1, dst,
441
+ ne00, ne01, ne02, ne03,
442
+ ne11, ne12,
443
+ nb01, nb02, nb03,
444
+ nb10, nb11, nb12,
445
+ nb1, nb2, nb3,
446
+ stream);
447
+ break;
448
+ case GGML_TYPE_Q5_K:
449
+ set_rows_sycl_qk_host<TIn, TIdx, block_q5_K, QK_K, quantize_row_q5_K_ref>(
450
+ src0, src1, dst,
451
+ ne00, ne01, ne02, ne03,
452
+ ne11, ne12,
453
+ nb01, nb02, nb03,
454
+ nb10, nb11, nb12,
455
+ nb1, nb2, nb3,
456
+ stream);
457
+ break;
458
+ case GGML_TYPE_Q6_K:
459
+ set_rows_sycl_qk_host<TIn, TIdx, block_q6_K, QK_K, quantize_row_q6_K_ref>(
460
+ src0, src1, dst,
461
+ ne00, ne01, ne02, ne03,
462
+ ne11, ne12,
463
+ nb01, nb02, nb03,
464
+ nb10, nb11, nb12,
465
+ nb1, nb2, nb3,
466
+ stream);
467
+ break;
468
+ case GGML_TYPE_IQ2_XXS:
469
+ set_rows_sycl_iq_host<TIn, TIdx, block_iq2_xxs, QK_K, quantize_iq2_xxs>(
470
+ src0, src1, dst,
471
+ ne00, ne01, ne02, ne03,
472
+ ne11, ne12,
473
+ nb01, nb02, nb03,
474
+ nb10, nb11, nb12,
475
+ nb1, nb2, nb3,
476
+ stream);
477
+ break;
478
+ case GGML_TYPE_IQ2_XS:
479
+ set_rows_sycl_iq_host<TIn, TIdx, block_iq2_xs, QK_K, quantize_iq2_xs>(
480
+ src0, src1, dst,
481
+ ne00, ne01, ne02, ne03,
482
+ ne11, ne12,
483
+ nb01, nb02, nb03,
484
+ nb10, nb11, nb12,
485
+ nb1, nb2, nb3,
486
+ stream);
487
+ break;
488
+ case GGML_TYPE_IQ2_S:
489
+ set_rows_sycl_iq_host<TIn, TIdx, block_iq2_s, QK_K, quantize_iq2_s>(
490
+ src0, src1, dst,
491
+ ne00, ne01, ne02, ne03,
492
+ ne11, ne12,
493
+ nb01, nb02, nb03,
494
+ nb10, nb11, nb12,
495
+ nb1, nb2, nb3,
496
+ stream);
497
+ break;
498
+ case GGML_TYPE_IQ3_XXS:
499
+ set_rows_sycl_qk_host<TIn, TIdx, block_iq3_xxs, QK_K, quantize_row_iq3_xxs_ref>(
500
+ src0, src1, dst,
501
+ ne00, ne01, ne02, ne03,
502
+ ne11, ne12,
503
+ nb01, nb02, nb03,
504
+ nb10, nb11, nb12,
505
+ nb1, nb2, nb3,
506
+ stream);
507
+ break;
508
+ case GGML_TYPE_IQ3_S:
509
+ set_rows_sycl_qk_host<TIn, TIdx, block_iq3_s, QK_K, quantize_row_iq3_s_ref>(
510
+ src0, src1, dst,
511
+ ne00, ne01, ne02, ne03,
512
+ ne11, ne12,
513
+ nb01, nb02, nb03,
514
+ nb10, nb11, nb12,
515
+ nb1, nb2, nb3,
516
+ stream);
517
+ break;
518
+ case GGML_TYPE_IQ1_S:
519
+ set_rows_sycl_iq_host<TIn, TIdx, block_iq1_s, QK_K, quantize_iq1_s>(
520
+ src0, src1, dst,
521
+ ne00, ne01, ne02, ne03,
522
+ ne11, ne12,
523
+ nb01, nb02, nb03,
524
+ nb10, nb11, nb12,
525
+ nb1, nb2, nb3,
526
+ stream);
527
+ break;
528
+ case GGML_TYPE_IQ1_M:
529
+ set_rows_sycl_iq_host<TIn, TIdx, block_iq1_m, QK_K, quantize_iq1_m>(
530
+ src0, src1, dst,
531
+ ne00, ne01, ne02, ne03,
532
+ ne11, ne12,
533
+ nb01, nb02, nb03,
534
+ nb10, nb11, nb12,
535
+ nb1, nb2, nb3,
536
+ stream);
537
+ break;
538
+ case GGML_TYPE_IQ4_XS:
539
+ set_rows_sycl_qk_host<TIn, TIdx, block_iq4_xs, QK_K, quantize_row_iq4_xs_ref>(
540
+ src0, src1, dst,
541
+ ne00, ne01, ne02, ne03,
542
+ ne11, ne12,
543
+ nb01, nb02, nb03,
544
+ nb10, nb11, nb12,
545
+ nb1, nb2, nb3,
546
+ stream);
547
+ break;
548
+ default:
549
+ GGML_ABORT("Unsupported tensor type: src0 %s src1 %s dst %s", ggml_type_name(dst->src[0]->type),
550
+ ggml_type_name(dst->src[1]->type), ggml_type_name(dst->type));
551
+ break;
552
+ }
553
+ }
554
+
555
+ void ggml_sycl_op_set_rows(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
556
+ scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
557
+ const ggml_tensor * src0 = dst->src[0];
558
+ const ggml_tensor * src1 = dst->src[1];
559
+
560
+ GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32 || dst->src[0]->type == GGML_TYPE_F16);
561
+ GGML_ASSERT(dst->src[1]->type == GGML_TYPE_I64 || dst->src[1]->type == GGML_TYPE_I32);
562
+
563
+ // dispatch on the index type (src1) and the source value type (src0)
564
+ if (src0->type == GGML_TYPE_F16) {
565
+ if (src1->type == GGML_TYPE_I64) {
566
+ set_rows_sycl<sycl::half, int64_t>(ctx, src0, src1, dst);
567
+ } else {
568
+ set_rows_sycl<sycl::half, int32_t>(ctx, src0, src1, dst);
569
+ }
570
+ } else {
571
+ if (src1->type == GGML_TYPE_I64) {
572
+ set_rows_sycl<float, int64_t>(ctx, src0, src1, dst);
573
+ } else {
574
+ set_rows_sycl<float, int32_t>(ctx, src0, src1, dst);
575
+ }
576
+ }
577
+ }
ggml/src/ggml-sycl/set_rows.hpp ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ #ifndef GGML_SYCL_SET_ROWS_HPP
2
+ #define GGML_SYCL_SET_ROWS_HPP
3
+
4
+ #include "common.hpp"
5
+
6
+ void ggml_sycl_op_set_rows(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
7
+
8
+ #endif // GGML_SYCL_SET_ROWS_HPP
ggml/src/ggml-sycl/softmax.cpp ADDED
@@ -0,0 +1,425 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #include "softmax.hpp"
2
+ #include <cstdint>
3
+ #include <utility>
4
+ #include <cmath>
5
+
6
+
7
+ template <typename T> static __dpct_inline__ float t2f32(T val) {
8
+ return (float) val;
9
+ }
10
+
11
+ template <> float __dpct_inline__ t2f32<sycl::half>(sycl::half val) {
12
+ return sycl::vec<sycl::half, 1>(val)
13
+ .convert<float, sycl::rounding_mode::automatic>()[0];
14
+ }
15
+
16
+ struct soft_max_params {
17
+
18
+ int64_t nheads;
19
+ uint32_t n_head_log2;
20
+ int64_t ncols;
21
+ int64_t nrows_x;
22
+ int64_t nrows_y;
23
+ int64_t ne00;
24
+ int64_t ne01;
25
+ int64_t ne02;
26
+ int64_t ne03;
27
+ int64_t nb11;
28
+ int64_t nb12;
29
+ int64_t nb13;
30
+
31
+ int64_t ne12;
32
+ int64_t ne13;
33
+ float scale;
34
+ float max_bias;
35
+ float m0;
36
+ float m1;
37
+ };
38
+
39
+ // When ncols_template == 0 the bounds for the loops in this function are not known and can't be unrolled.
40
+ // As we want to keep pragma unroll for all other cases we suppress the clang transformation warning here.
41
+ #ifdef __clang__
42
+ #pragma clang diagnostic push
43
+ #pragma clang diagnostic ignored "-Wpass-failed"
44
+ #endif // __clang__
45
+ template <bool use_shared, int ncols_template, int block_size_template, typename T>
46
+ static void soft_max_f32(const float * x,
47
+ const T * mask,
48
+ const float * sinks,
49
+ float * dst,
50
+ const soft_max_params p,
51
+ uint8_t * dpct_local) {
52
+ auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
53
+ const int ncols = ncols_template == 0 ? p.ncols : ncols_template;
54
+ const int block_size = block_size_template == 0
55
+ ? item_ct1.get_local_range(2)
56
+ : block_size_template;
57
+ const int nthreads = block_size;
58
+ const int nwarps = nthreads / WARP_SIZE;
59
+ const size_t nreduce = nwarps / WARP_SIZE;
60
+
61
+ const int tid = item_ct1.get_local_id(2);
62
+
63
+ const int64_t i03 = item_ct1.get_group(0);
64
+ const int64_t i02 = item_ct1.get_group(1);
65
+ const int64_t i01 = item_ct1.get_group(2);
66
+
67
+ //TODO: noncontigous inputs/outputs
68
+ const int rowx = item_ct1.get_group(2) +
69
+ item_ct1.get_group(1) * item_ct1.get_group_range(2) +
70
+ item_ct1.get_group(0) * item_ct1.get_group_range(2) *
71
+ item_ct1.get_group_range(1);
72
+
73
+ const int64_t i11 = i01;
74
+ const int64_t i12 = i02 % p.ne12;
75
+ const int64_t i13 = i03 % p.ne13;
76
+
77
+ x += int64_t(rowx)*ncols;
78
+ mask += (i11*p.nb11 + i12*p.nb12 + i13*p.nb13) / sizeof(T) * (mask != nullptr);
79
+ dst += int64_t(rowx)*ncols;
80
+
81
+ const int warp_id = item_ct1.get_local_id(2) / WARP_SIZE;
82
+ const int lane_id = item_ct1.get_local_id(2) % WARP_SIZE;
83
+
84
+ const float slope = get_alibi_slope(p.max_bias, i02, p.n_head_log2, p.m0, p.m1);
85
+
86
+ float * buf_iw = (float *) dpct_local;
87
+
88
+ // shared memory buffer to cache values between iterations:
89
+ float *vals = use_shared ? buf_iw + sycl::max(nwarps, WARP_SIZE) : dst;
90
+ float max_val = sinks ? sinks[i02] : -INFINITY;
91
+ #pragma unroll
92
+ for (int col0 = 0; col0 < ncols; col0 += block_size) {
93
+ const int col = col0 + tid;
94
+
95
+ if (ncols_template == 0 && col >= ncols) {
96
+ break;
97
+ }
98
+
99
+ const float val = x[col]*p.scale + (mask ? slope*t2f32(mask[col]) : 0.0f);
100
+
101
+ vals[col] = val;
102
+ max_val = sycl::max(max_val, val);
103
+ }
104
+ // find the max value in the block
105
+ max_val = warp_reduce_max<WARP_SIZE>(max_val);
106
+
107
+ if (block_size > WARP_SIZE) {
108
+ if (lane_id == 0) {
109
+ buf_iw[warp_id] = max_val;
110
+ }
111
+ item_ct1.barrier();
112
+
113
+ max_val = -INFINITY;
114
+ for (int i = lane_id; i < nwarps; i += WARP_SIZE) {
115
+ max_val = sycl::max(max_val, buf_iw[i]);
116
+ }
117
+ max_val = warp_reduce_max<WARP_SIZE>(max_val);
118
+ }
119
+ float tmp = 0.0f; // partial sum
120
+
121
+ #pragma unroll
122
+ for (int col0 = 0; col0 < ncols; col0 += block_size) {
123
+ const int col = col0 + tid;
124
+
125
+ if (ncols_template == 0 && col >= ncols) {
126
+ break;
127
+ }
128
+
129
+ const float val = sycl::native::exp(sycl::max(vals[col] - max_val, -80.0f));
130
+ tmp += val;
131
+ vals[col] = val;
132
+ }
133
+ // find the sum of exps in the block
134
+ tmp = warp_reduce_sum<WARP_SIZE>(tmp);
135
+ if (block_size > WARP_SIZE) {
136
+ item_ct1.barrier();
137
+ if (warp_id == 0) {
138
+ buf_iw[lane_id] = 0.0f;
139
+ for (size_t i = 1; i < nreduce; i += 1) {
140
+ buf_iw[lane_id + i * WARP_SIZE] = 0.f;
141
+ }
142
+ }
143
+ item_ct1.barrier();
144
+
145
+ if (lane_id == 0) {
146
+ buf_iw[warp_id] = tmp;
147
+ }
148
+ item_ct1.barrier();
149
+
150
+ tmp = buf_iw[lane_id];
151
+ for (size_t i = 1; i < nreduce; i += 1) {
152
+ tmp += buf_iw[lane_id + i * WARP_SIZE];
153
+ }
154
+ tmp = warp_reduce_sum<WARP_SIZE>(tmp);
155
+ }
156
+ if (sinks) {
157
+ tmp += sycl::native::exp(sycl::max(sinks[i02] - max_val, -80.0f));
158
+ }
159
+ const float inv_sum = 1.0f / tmp;
160
+
161
+ #pragma unroll
162
+ for (int col0 = 0; col0 < ncols; col0 += block_size) {
163
+ const int col = col0 + tid;
164
+
165
+ if (ncols_template == 0 && col >= ncols) {
166
+ return;
167
+ }
168
+
169
+ dst[col] = vals[col] * inv_sum;
170
+ }
171
+ }
172
+ #ifdef __clang__
173
+ #pragma clang diagnostic pop
174
+ #endif // __clang__
175
+
176
+ static void soft_max_back_f32(const float *grad, const float *dstf, float *dst,
177
+ const int ncols, const float scale) {
178
+ auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
179
+ const int tid = item_ct1.get_local_id(2);
180
+ const int rowx = item_ct1.get_group(2);
181
+
182
+ grad += int64_t(rowx)*ncols;
183
+ dstf += int64_t(rowx)*ncols;
184
+ dst += int64_t(rowx)*ncols;
185
+
186
+ float dgf_dot = 0.0f; // dot product of dst from forward pass and gradients
187
+
188
+ for (int col = tid; col < ncols; col += WARP_SIZE) {
189
+ dgf_dot += dstf[col]*grad[col];
190
+ }
191
+
192
+ dgf_dot = warp_reduce_sum<WARP_SIZE>(dgf_dot);
193
+
194
+ for (int col = tid; col < ncols; col += WARP_SIZE) {
195
+ dst[col] = scale * (grad[col] - dgf_dot) * dstf[col];
196
+ }
197
+ }
198
+
199
+ template <int... Ns, typename T>
200
+ static void launch_soft_max_kernels(const float * x,
201
+ const T * mask,
202
+ const float * sinks,
203
+ float * dst,
204
+ const soft_max_params & p,
205
+ dpct::queue_ptr stream,
206
+ dpct::dim3 block_dims,
207
+ dpct::dim3 block_nums,
208
+ size_t nbytes_shared)
209
+ {
210
+ auto launch_kernel = [=](auto I) -> bool {
211
+ constexpr int ncols = decltype(I)::value;
212
+ constexpr int block = (ncols > 1024 ? 1024 : ncols);
213
+ if (p.ncols == ncols) {
214
+ stream->submit([&](sycl::handler &cgh) {
215
+ sycl::local_accessor<uint8_t, 1> dpct_local_acc_ct1(
216
+ sycl::range<1>(nbytes_shared), cgh);
217
+
218
+ cgh.parallel_for(
219
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
220
+ [=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(
221
+ WARP_SIZE)]] {
222
+ soft_max_f32<true, ncols, block>(
223
+ x, mask, sinks, dst, p,
224
+ dpct_local_acc_ct1
225
+ .get_multi_ptr<sycl::access::decorated::no>()
226
+ .get());
227
+ GGML_UNUSED(item_ct1);
228
+ });
229
+ });
230
+ return true;
231
+ }
232
+ return false;
233
+ };
234
+
235
+ // unary fold over launch_kernel
236
+ if ((launch_kernel(std::integral_constant<int, Ns>{}) || ...)) {
237
+ return;
238
+ }
239
+
240
+ stream->submit([&](sycl::handler &cgh) {
241
+ sycl::local_accessor<uint8_t, 1> dpct_local_acc_ct1(
242
+ sycl::range<1>(nbytes_shared), cgh);
243
+
244
+ cgh.parallel_for(
245
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
246
+ [=](sycl::nd_item<3> item_ct1)
247
+ [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
248
+ soft_max_f32<true, 0, 0>(
249
+ x, mask, sinks, dst, p,
250
+ dpct_local_acc_ct1
251
+ .get_multi_ptr<sycl::access::decorated::no>()
252
+ .get());
253
+ GGML_UNUSED(item_ct1);
254
+ });
255
+ });
256
+ }
257
+
258
+ template <typename T>
259
+ static void soft_max_f32_sycl(const float *x, const T *mask,
260
+ const float *sinks, float *dst,
261
+ const soft_max_params &params,
262
+ dpct::queue_ptr stream, int device) {
263
+ int nth = WARP_SIZE;
264
+ int max_block_size = ggml_sycl_info().max_work_group_sizes[device];
265
+ const int64_t ncols_x = params.ncols;
266
+
267
+ while (nth < ncols_x && nth < max_block_size) nth *= 2;
268
+ if (nth>max_block_size) nth = max_block_size;
269
+
270
+ const dpct::dim3 block_dims(nth, 1, 1);
271
+ const dpct::dim3 block_nums(params.ne01, params.ne02, params.ne03);
272
+ const size_t nbytes_shared =
273
+ (GGML_PAD(ncols_x, WARP_SIZE) + WARP_SIZE) * sizeof(float);
274
+
275
+ const int id = get_current_device_id();
276
+ const size_t smpbo = ggml_sycl_info().devices[id].smpbo;
277
+
278
+ if (nbytes_shared <= smpbo && ncols_x <= max_block_size) {
279
+ launch_soft_max_kernels<32, 64, 128, 256, 512, 1024, 2048, 4096>(
280
+ x, mask, sinks, dst, params, stream, block_dims, block_nums,
281
+ nbytes_shared);
282
+ } else {
283
+ const size_t nbytes_shared_low = WARP_SIZE * sizeof(float);
284
+
285
+ stream->submit([&](sycl::handler &cgh) {
286
+ sycl::local_accessor<uint8_t, 1> dpct_local_acc_ct1(
287
+ sycl::range<1>(nbytes_shared_low), cgh);
288
+
289
+ cgh.parallel_for(
290
+ sycl::nd_range<3>(block_nums * block_dims, block_dims),
291
+ [=](sycl::nd_item<3> item_ct1)
292
+ [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
293
+ soft_max_f32<false, 0, 0>(
294
+ x, mask, sinks, dst, params,
295
+ dpct_local_acc_ct1
296
+ .get_multi_ptr<sycl::access::decorated::no>()
297
+ .get());
298
+ GGML_UNUSED(item_ct1);
299
+ });
300
+ });
301
+ }
302
+ }
303
+
304
+ static void soft_max_back_f32_sycl(const float * grad,
305
+ const float * dstf,
306
+ float * dst,
307
+ const int ncols,
308
+ const int nrows,
309
+ const float scale,
310
+ dpct::queue_ptr stream) {
311
+ const dpct::dim3 block_dims(WARP_SIZE, 1, 1);
312
+ const dpct::dim3 block_nums(nrows, 1, 1);
313
+
314
+ stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
315
+ [=](sycl::nd_item<3> item_ct1) {
316
+ soft_max_back_f32(grad, dstf, dst, ncols, scale);
317
+ GGML_UNUSED(item_ct1);
318
+ });
319
+ }
320
+
321
+ void ggml_sycl_op_soft_max(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
322
+ scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
323
+
324
+ const ggml_tensor * src0 = dst->src[0];
325
+ const ggml_tensor * src1 = dst->src[1];
326
+ const ggml_tensor * src2 = dst->src[2];
327
+
328
+ const float * src0_d = (const float *) src0->data;
329
+ const void * src1_d = src1 ? (const void *) src1->data : nullptr;
330
+ const void * src2_d = src2 ? (const void *) src2->data : nullptr;
331
+ float * dst_d = (float *) dst->data;
332
+
333
+ dpct::queue_ptr stream = ctx.stream();
334
+
335
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
336
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
337
+
338
+ // src1 contains mask and it is optional
339
+ GGML_ASSERT(!src1 || src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_F32);
340
+
341
+ const int64_t nrows_x = ggml_nrows(src0);
342
+ const int64_t nrows_y = src0->ne[1];
343
+
344
+ const int64_t ne00 = src0->ne[0];
345
+
346
+ float scale = 1.0f;
347
+ float max_bias = 0.0f;
348
+
349
+ memcpy(&scale, (const float *) dst->op_params + 0, sizeof(float));
350
+ memcpy(&max_bias, (const float *) dst->op_params + 1, sizeof(float));
351
+
352
+ const bool use_f16 = (src1 && src1->type == GGML_TYPE_F16);
353
+
354
+ const int64_t nb11 = src1 ? src1->nb[1] : 1;
355
+ const int64_t nb12 = src1 ? src1->nb[2] : 1;
356
+ const int64_t nb13 = src1 ? src1->nb[3] : 1;
357
+
358
+ const int64_t ne12 = src1 ? src1->ne[2] : 1;
359
+ const int64_t ne13 = src1 ? src1->ne[3] : 1;
360
+
361
+ const uint32_t n_head = src0->ne[2];
362
+ const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head));
363
+
364
+ const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);
365
+ const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);
366
+
367
+
368
+ soft_max_params params = {};
369
+ params.nheads = src0->ne[2];
370
+ params.n_head_log2 = n_head_log2;
371
+ params.ncols = ne00;
372
+ params.nrows_x = nrows_x;
373
+ params.nrows_y = nrows_y;
374
+ params.ne00 = src0->ne[0];
375
+ params.ne01 = src0->ne[1];
376
+ params.ne02 = src0->ne[2];
377
+ params.ne03 = src0->ne[3];
378
+ params.nb11 = nb11;
379
+ params.nb12 = nb12;
380
+ params.nb13 = nb13;
381
+ params.ne12 = ne12;
382
+ params.ne13 = ne13;
383
+ params.scale = scale;
384
+ params.max_bias = max_bias;
385
+ params.m0 = m0;
386
+ params.m1 = m1;
387
+
388
+ if (use_f16) {
389
+ soft_max_f32_sycl(src0_d, (const sycl::half *)src1_d,
390
+ (const float *)src2_d, dst_d, params, stream,
391
+ ctx.device);
392
+ } else {
393
+ soft_max_f32_sycl(src0_d, (const float *)src1_d, (const float *)src2_d,
394
+ dst_d, params, stream, ctx.device);
395
+ }
396
+ }
397
+
398
+ void ggml_sycl_op_soft_max_back(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
399
+ scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
400
+ const ggml_tensor * src0 = dst->src[0]; // grad
401
+ const ggml_tensor * src1 = dst->src[1]; // forward pass output
402
+
403
+ const float * src0_d = (const float *) src0->data;
404
+ const float * src1_d = (const float *) src1->data;
405
+ float * dst_d = (float *) dst->data;
406
+
407
+ dpct::queue_ptr stream = ctx.stream();
408
+
409
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
410
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
411
+ GGML_ASSERT( dst->type == GGML_TYPE_F32);
412
+
413
+ const int64_t ncols = src0->ne[0];
414
+ const int64_t nrows = ggml_nrows(src0);
415
+
416
+ float scale = 1.0f;
417
+ float max_bias = 0.0f;
418
+
419
+ memcpy(&scale, (const float *) dst->op_params + 0, sizeof(float));
420
+ memcpy(&max_bias, (const float *) dst->op_params + 1, sizeof(float));
421
+
422
+ GGML_ASSERT(max_bias == 0.0f);
423
+
424
+ soft_max_back_f32_sycl(src0_d, src1_d, dst_d, ncols, nrows, scale, stream);
425
+ }
ggml/src/ggml-sycl/softmax.hpp ADDED
@@ -0,0 +1,24 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //
2
+ // MIT license
3
+ // Copyright (C) 2024 Intel Corporation
4
+ // SPDX-License-Identifier: MIT
5
+ //
6
+
7
+ //
8
+ // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
9
+ // See https://llvm.org/LICENSE.txt for license information.
10
+ // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
11
+ //
12
+
13
+ #ifndef GGML_SYCL_SOFTMAX_HPP
14
+ #define GGML_SYCL_SOFTMAX_HPP
15
+
16
+ #include "common.hpp"
17
+
18
+ #define SYCL_SOFT_MAX_BLOCK_SIZE 1024
19
+
20
+ void ggml_sycl_op_soft_max(ggml_backend_sycl_context &ctx, ggml_tensor *dst);
21
+
22
+ void ggml_sycl_op_soft_max_back(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
23
+
24
+ #endif // GGML_SYCL_SOFTMAX_HPP
ggml/src/ggml-sycl/solve_tri.cpp ADDED
@@ -0,0 +1,172 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #include "solve_tri.hpp"
2
+ #include "common.hpp"
3
+ #include <oneapi/mkl/blas.hpp>
4
+
5
+ template <int n_template, int k_template>
6
+ static void solve_tri_f32_fast(const float * __restrict__ A,
7
+ const float * __restrict__ B,
8
+ float * __restrict__ X,
9
+ const int64_t ne02, [[maybe_unused]] const int64_t ne03,
10
+ const int64_t nb02, const int64_t nb03,
11
+ const int64_t nb12, const int64_t nb13,
12
+ const int64_t nb2, const int64_t nb3,
13
+ const int n_arg, const int k_arg,
14
+ const sycl::nd_item<2> & item, float * sA) {
15
+
16
+ const int n = n_template == 0 ? n_arg : n_template;
17
+ const int k = k_template == 0 ? k_arg : k_template;
18
+
19
+ const int batch_idx = item.get_group(1);
20
+ const int lane = item.get_local_id(1) % WARP_SIZE;
21
+ const int col_idx = item.get_local_id(0);
22
+
23
+ if (col_idx >= k) {
24
+ return;
25
+ }
26
+
27
+ const int64_t i03 = batch_idx / ne02;
28
+ const int64_t i02 = batch_idx % ne02;
29
+
30
+ const float * A_batch = (const float *) ((const char *) A + i02 * nb02 + i03 * nb03);
31
+ const float * B_batch = (const float *) ((const char *) B + i02 * nb12 + i03 * nb13);
32
+ float * X_batch = (float *) ((char *) X + i02 * nb2 + i03 * nb3);
33
+
34
+ const int offset = item.get_local_id(1) + item.get_local_id(0) * item.get_local_range(1);
35
+
36
+ #pragma unroll
37
+ for (int i = 0; i < n * n; i += k * WARP_SIZE) {
38
+ const int i0 = i + offset;
39
+ if (i0 < n * n) {
40
+ sA[i0] = A_batch[i0];
41
+ }
42
+ }
43
+
44
+ item.barrier(sycl::access::fence_space::local_space);
45
+
46
+ float x_low = (lane < n) ? B_batch[lane * k + col_idx] : 0.0f;
47
+ float x_high = (WARP_SIZE + lane < n) ? B_batch[(WARP_SIZE + lane) * k + col_idx] : 0.0f;
48
+
49
+ const int half = WARP_SIZE;
50
+ const int nrows_low = (n < half) ? n : half;
51
+
52
+ #pragma unroll
53
+ for (int row = 0; row < nrows_low; ++row) {
54
+ float sum = 0.0f;
55
+ if (lane < row) {
56
+ sum += sA[row * n + lane] * x_low;
57
+ }
58
+ sum = warp_reduce_sum<WARP_SIZE>(sum);
59
+ if (lane == row) {
60
+ x_low = (x_low - sum) / sA[row * n + row];
61
+ }
62
+ }
63
+
64
+ #pragma unroll
65
+ for (int row = half; row < n; ++row) {
66
+ float sum = sA[row * n + lane] * x_low;
67
+ const int j = half + lane;
68
+ if (j < row) {
69
+ sum += sA[row * n + j] * x_high;
70
+ }
71
+ sum = warp_reduce_sum<WARP_SIZE>(sum);
72
+ if (lane == row - half) {
73
+ x_high = (x_high - sum) / sA[row * n + row];
74
+ }
75
+ }
76
+
77
+ #pragma unroll
78
+ for (int rr = 0; rr < 2; ++rr) {
79
+ const int row = rr * WARP_SIZE + lane;
80
+ if (row < n) {
81
+ const float val = (row < half) ? x_low : x_high;
82
+ X_batch[row * k + col_idx] = val;
83
+ }
84
+ }
85
+ }
86
+
87
+ static void solve_tri_f32_mkl(dpct::queue_ptr stream,
88
+ const float * A, float * X,
89
+ int n, int k,
90
+ int64_t ne02, [[maybe_unused]] int64_t ne03,
91
+ int64_t nb02, [[maybe_unused]] int64_t nb03,
92
+ int64_t nb2, [[maybe_unused]] int64_t nb3) {
93
+ const float alpha = 1.0f;
94
+ const int64_t total_batches = ne02 * ne03;
95
+ if (total_batches == 0) {
96
+ return;
97
+ }
98
+
99
+ const int64_t stride_a = nb02 / sizeof(float);
100
+ const int64_t stride_x = nb2 / sizeof(float);
101
+
102
+ oneapi::mkl::blas::trsm_batch(
103
+ *stream,
104
+ oneapi::mkl::side::right,
105
+ oneapi::mkl::uplo::upper,
106
+ oneapi::mkl::transpose::nontrans,
107
+ oneapi::mkl::diag::nonunit,
108
+ k, n, alpha,
109
+ A, n, stride_a,
110
+ X, k, stride_x,
111
+ total_batches);
112
+ }
113
+
114
+ inline void ggml_sycl_op_solve_tri(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
115
+ const ggml_tensor * src0 = dst->src[0];
116
+ const ggml_tensor * src1 = dst->src[1];
117
+
118
+ GGML_ASSERT(ggml_is_contiguous(src0));
119
+ GGML_ASSERT(ggml_is_contiguous(src1));
120
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
121
+
122
+ dpct::queue_ptr stream = ctx.stream();
123
+ SYCL_CHECK(ggml_sycl_set_device(ctx.device));
124
+
125
+ const int n = src0->ne[0];
126
+ const int k = src1->ne[0];
127
+ const int64_t ne02 = src0->ne[2];
128
+ const int64_t ne03 = src0->ne[3];
129
+
130
+ GGML_ASSERT(n <= SYCL_SOLVE_TRI_MAX_N && k <= SYCL_SOLVE_TRI_MAX_K);
131
+
132
+ const float * A_d = static_cast<const float *>(src0->data);
133
+ const float * B_d = static_cast<const float *>(src1->data);
134
+ float * X_d = static_cast<float *>(dst->data);
135
+
136
+ if (X_d != B_d) {
137
+ const int64_t total_elements = (int64_t)n * k * ne02 * ne03;
138
+ stream->memcpy(X_d, B_d, total_elements * sizeof(float));
139
+ }
140
+
141
+ const int64_t nb02 = src0->nb[2];
142
+ const int64_t nb03 = src0->nb[3];
143
+ const int64_t nb12 = src1->nb[2];
144
+ const int64_t nb13 = src1->nb[3];
145
+ const int64_t nb2 = dst->nb[2];
146
+ const int64_t nb3 = dst->nb[3];
147
+
148
+ const int64_t total_batches = ne02 * ne03;
149
+
150
+ if (n <= 2 * WARP_SIZE && k <= 32) {
151
+ const int smem_size = 2 * WARP_SIZE * 2 * WARP_SIZE;
152
+ const sycl::range<2> grid(1, total_batches);
153
+ const sycl::range<2> block(k, WARP_SIZE);
154
+ stream->submit([&](sycl::handler & cgh) {
155
+ sycl::local_accessor<float, 1> smem_acc(sycl::range<1>(smem_size), cgh);
156
+ cgh.parallel_for(
157
+ sycl::nd_range<2>(grid * block, block),
158
+ [=](sycl::nd_item<2> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
159
+ solve_tri_f32_fast<0, 0>(A_d, B_d, X_d, ne02, ne03,
160
+ nb02, nb03, nb12, nb13, nb2, nb3,
161
+ n, k, item, get_pointer(smem_acc));
162
+ });
163
+ });
164
+ } else {
165
+ solve_tri_f32_mkl(stream, A_d, X_d, n, k, ne02, ne03, nb02, nb03, nb2, nb3);
166
+ }
167
+ }
168
+
169
+ void ggml_sycl_solve_tri(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
170
+ scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
171
+ ggml_sycl_op_solve_tri(ctx, dst);
172
+ }
ggml/src/ggml-sycl/solve_tri.hpp ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ #pragma once
2
+
3
+ #include "common.hpp"
4
+
5
+ #define SYCL_SOLVE_TRI_MAX_N 64
6
+ #define SYCL_SOLVE_TRI_MAX_K 64
7
+
8
+ void ggml_sycl_solve_tri(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
ggml/src/ggml-sycl/ssm_conv.cpp ADDED
@@ -0,0 +1,136 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #include "ssm_conv.hpp"
2
+ #include "common.hpp"
3
+
4
+ #include <cstdio>
5
+
6
+ using namespace sycl;
7
+
8
+ static void kernel_ssm_conv(
9
+ queue &q,
10
+ const float *src_data,
11
+ const float *weights,
12
+ float *dst_data,
13
+ int d_conv,
14
+ int d_inner,
15
+ int n_t,
16
+ int n_s,
17
+ int ncs __attribute__((unused)),
18
+ int src_stride_inner,
19
+ int src_stride_seq,
20
+ int dst_stride_token,
21
+ int dst_stride_seq
22
+ ) {
23
+ const size_t total_work = static_cast<size_t>(d_inner) * static_cast<size_t>(n_t) * static_cast<size_t>(n_s);
24
+ const size_t work_group_size = 256;
25
+ const size_t num_work_groups = (total_work + work_group_size - 1) / work_group_size;
26
+
27
+ const range<1> global_range(num_work_groups * work_group_size);
28
+ const range<1> local_range(work_group_size);
29
+
30
+ q.submit([&](handler &h) {
31
+ h.parallel_for(
32
+ nd_range<1>(global_range, local_range),
33
+ [=](nd_item<1> item) {
34
+ const size_t idx = item.get_global_id(0);
35
+ if (idx >= total_work) {
36
+ return;
37
+ }
38
+
39
+ // src has the tokens of one channel contiguous, dst has the channels of one
40
+ // token contiguous, so either the loads or the store must be strided. Indexing
41
+ // token-fastest coalesces the d_conv loads, which measured faster except for
42
+ // short, cache-resident rows.
43
+ const int token = static_cast<int>(idx % n_t);
44
+ const int channel = static_cast<int>((idx / n_t) % d_inner);
45
+ const int seq = static_cast<int>(idx / (static_cast<size_t>(n_t) * static_cast<size_t>(d_inner)));
46
+
47
+ const float *s = src_data
48
+ + static_cast<size_t>(seq) * static_cast<size_t>(src_stride_seq)
49
+ + static_cast<size_t>(channel) * static_cast<size_t>(src_stride_inner)
50
+ + static_cast<size_t>(token);
51
+
52
+ const float *c = weights + static_cast<size_t>(channel) * static_cast<size_t>(d_conv);
53
+
54
+ float sumf = 0.0f;
55
+ for (int i0 = 0; i0 < d_conv; ++i0) {
56
+ sumf += s[i0] * c[i0];
57
+ }
58
+
59
+ const size_t dst_idx =
60
+ static_cast<size_t>(seq) * static_cast<size_t>(dst_stride_seq) +
61
+ static_cast<size_t>(token) * static_cast<size_t>(dst_stride_token) +
62
+ static_cast<size_t>(channel);
63
+
64
+ dst_data[dst_idx] = sumf;
65
+ }
66
+ );
67
+ });
68
+ }
69
+
70
+ inline void ggml_sycl_op_ssm_conv(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
71
+ ggml_tensor * src0 = dst->src[0];
72
+ ggml_tensor * src1 = dst->src[1];
73
+
74
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
75
+ GGML_ASSERT(src1->type == GGML_TYPE_F32);
76
+ GGML_ASSERT(dst->type == GGML_TYPE_F32);
77
+
78
+ const int d_conv = src1->ne[0];
79
+ const int ncs = src0->ne[0];
80
+ const int d_inner = src0->ne[1];
81
+ const int n_t = dst->ne[1];
82
+ const int n_s = dst->ne[2];
83
+
84
+ GGML_ASSERT(src0->ne[0] == d_conv - 1 + n_t);
85
+ GGML_ASSERT(src0->ne[1] == d_inner);
86
+ GGML_ASSERT(src1->ne[1] == d_inner);
87
+
88
+ GGML_ASSERT(dst->ne[0] == d_inner);
89
+ GGML_ASSERT(dst->ne[1] == n_t);
90
+ GGML_ASSERT(dst->ne[2] == n_s);
91
+
92
+ GGML_ASSERT(src0->nb[0] == sizeof(float));
93
+ GGML_ASSERT(src1->nb[0] == sizeof(float));
94
+
95
+ GGML_ASSERT(src0->nb[1] == src0->ne[0] * sizeof(float));
96
+
97
+ const int src_stride_inner = ncs;
98
+ const int src_stride_seq = ncs * d_inner;
99
+ const int dst_stride_token = d_inner;
100
+ const int dst_stride_seq = d_inner * n_t;
101
+
102
+ try {
103
+ queue *q = ctx.stream();
104
+
105
+ const float *src_data = static_cast<const float *>(src0->data);
106
+ const float *weights = static_cast<const float *>(src1->data);
107
+ float *dst_data = static_cast<float *>(dst->data);
108
+
109
+ GGML_ASSERT(src_data && weights && dst_data);
110
+
111
+ kernel_ssm_conv(
112
+ *q,
113
+ src_data,
114
+ weights,
115
+ dst_data,
116
+ d_conv,
117
+ d_inner,
118
+ n_t,
119
+ n_s,
120
+ ncs,
121
+ src_stride_inner,
122
+ src_stride_seq,
123
+ dst_stride_token,
124
+ dst_stride_seq
125
+ );
126
+
127
+ } catch (const std::exception &e) {
128
+ std::fprintf(stderr, "[SYCL-SSM_CONV] ERROR: %s\n", e.what());
129
+ throw;
130
+ }
131
+ }
132
+
133
+ void ggml_sycl_ssm_conv(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
134
+ scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
135
+ ggml_sycl_op_ssm_conv(ctx, dst);
136
+ }
ggml/src/ggml-sycl/ssm_conv.hpp ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ #pragma once
2
+
3
+ #include "common.hpp"
4
+
5
+ void ggml_sycl_ssm_conv(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
ggml/src/ggml-sycl/ssm_scan.cpp ADDED
@@ -0,0 +1,170 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #include "ssm_scan.hpp"
2
+ #include "common.hpp"
3
+
4
+ template <int c_factor, int d_state>
5
+ static void ssm_scan_f32_group(
6
+ const float * __restrict__ src0, const float * __restrict__ src1, const float * __restrict__ src2,
7
+ const float * __restrict__ src3, const float * __restrict__ src4, const float * __restrict__ src5,
8
+ const int32_t * __restrict__ src6, float * __restrict__ dst,
9
+ const int src0_nb2, const int src0_nb3, const int src1_nb2, const int src1_nb3,
10
+ const int src2_nb1, const int src2_nb2, const int src3_nb1,
11
+ const int src4_nb2, const int src4_nb3, const int src5_nb2, const int src5_nb3,
12
+ const int64_t s_off, const int64_t n_head, const int64_t d_head, const int64_t n_group, const int64_t n_tok,
13
+ const int64_t K,
14
+ const sycl::nd_item<2> & item) {
15
+
16
+ const int lane = item.get_local_id(1) % WARP_SIZE;
17
+ const int warp = item.get_local_id(1) / WARP_SIZE;
18
+ const int warp_idx = item.get_group(1) * c_factor + warp;
19
+ const int seq_idx = item.get_group(0);
20
+
21
+ const int head_idx = warp_idx / d_head;
22
+ const int head_off = (warp_idx % d_head) * sizeof(float);
23
+ const int group_off = (head_idx / (n_head / n_group)) * d_state * sizeof(float);
24
+
25
+ const float * s0_warp = (const float *) ((const char *) src0 + src6[seq_idx] * src0_nb3 + head_idx * src0_nb2 + head_off * d_state);
26
+ const float * x_warp = (const float *) ((const char *) src1 + (seq_idx * src1_nb3) + (warp_idx * sizeof(float)));
27
+ const float * dt_warp = (const float *) ((const char *) src2 + (seq_idx * src2_nb2) + head_idx * sizeof(float));
28
+ const float * A_warp = (const float *) ((const char *) src3 + head_idx * src3_nb1);
29
+ const float * B_warp = (const float *) ((const char *) src4 + (seq_idx * src4_nb3) + (group_off));
30
+ const float * C_warp = (const float *) ((const char *) src5 + (seq_idx * src5_nb3) + (group_off));
31
+ float * y_warp = dst + (seq_idx * n_tok * n_head * d_head) + warp_idx;
32
+ float * s_warp = (float *) ((char *) dst + s_off + seq_idx * src0_nb3 + head_idx * src0_nb2 + head_off * d_state);
33
+
34
+ const int stride_x = src1_nb2 / sizeof(float);
35
+ const int stride_dt = src2_nb1 / sizeof(float);
36
+ const int stride_B = src4_nb2 / sizeof(float);
37
+ const int stride_C = src5_nb2 / sizeof(float);
38
+ const int stride_y = n_head * d_head;
39
+
40
+ float state[c_factor];
41
+ float state_sum = 0.0f;
42
+
43
+ #pragma unroll
44
+ for (int j = 0; j < c_factor; j++) {
45
+ state[j] = s0_warp[WARP_SIZE * j + lane];
46
+ }
47
+
48
+ for (int64_t i = 0; i < n_tok; i++) {
49
+ const float dt_val = dt_warp[i * stride_dt];
50
+ const float dt_soft_plus = (dt_val <= 20.0f ? sycl::log1p(sycl::exp(dt_val)) : dt_val);
51
+
52
+ state_sum = 0.0f;
53
+ const float dA = sycl::exp(dt_soft_plus * A_warp[0]);
54
+ const float x_dt = x_warp[i * stride_x] * dt_soft_plus;
55
+ #pragma unroll
56
+ for (int j = 0; j < c_factor; j++) {
57
+ const float B_val = B_warp[i * stride_B + WARP_SIZE * j + lane];
58
+ const float C_val = C_warp[i * stride_C + WARP_SIZE * j + lane];
59
+ state[j] = (state[j] * dA) + (B_val * x_dt);
60
+ state_sum += state[j] * C_val;
61
+ }
62
+
63
+ state_sum = warp_reduce_sum<WARP_SIZE>(state_sum);
64
+
65
+ if (lane == 0) {
66
+ y_warp[i * stride_y] = state_sum;
67
+ }
68
+
69
+ const int64_t slot = n_tok - 1 - i;
70
+ if (K > 1 && slot > 0 && slot < K) {
71
+ float * s_snapshot_warp = (float *) ((char *) dst + s_off + (slot * item.get_group_range(0) + seq_idx) * src0_nb3 + head_idx * src0_nb2 + head_off * d_state);
72
+ #pragma unroll
73
+ for (int j = 0; j < c_factor; j++) {
74
+ s_snapshot_warp[WARP_SIZE * j + lane] = state[j];
75
+ }
76
+ }
77
+ }
78
+
79
+ #pragma unroll
80
+ for (int j = 0; j < c_factor; j++) {
81
+ s_warp[WARP_SIZE * j + lane] = state[j];
82
+ }
83
+ }
84
+
85
+ static void ssm_scan_f32_sycl(
86
+ const float * src0, const float * src1, const float * src2, const float * src3,
87
+ const float * src4, const float * src5, const int32_t * src6, float * dst,
88
+ const int src0_nb2, const int src0_nb3, const int src1_nb2, const int src1_nb3, const int src2_nb1,
89
+ const int src2_nb2, const int src3_nb1, const int src4_nb2, const int src4_nb3, const int src5_nb2,
90
+ const int src5_nb3, const int64_t s_off, const int64_t d_state, const int64_t head_dim,
91
+ const int64_t n_head, const int64_t n_group, const int64_t n_tok, const int64_t n_seq,
92
+ const int64_t K,
93
+ dpct::queue_ptr stream) {
94
+
95
+ // NOTE: if you change conditions here, be sure to update the corresponding supports_op condition!
96
+ GGML_ASSERT(src3_nb1 == sizeof(float));
97
+ if (d_state == 128) {
98
+ constexpr int threads = 128;
99
+ constexpr int num_warps = threads / WARP_SIZE;
100
+ const sycl::range<2> grid(n_seq, (n_head * head_dim + num_warps - 1) / num_warps);
101
+ const sycl::range<2> block(1, threads);
102
+ stream->parallel_for(
103
+ sycl::nd_range<2>(grid * block, block),
104
+ [=](sycl::nd_item<2> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
105
+ ssm_scan_f32_group<128 / WARP_SIZE, 128>(
106
+ src0, src1, src2, src3, src4, src5, src6, dst,
107
+ src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2, src3_nb1,
108
+ src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, head_dim, n_group, n_tok, K, item);
109
+ });
110
+ } else if (d_state == 256) {
111
+ constexpr int threads = 256;
112
+ constexpr int num_warps = threads / WARP_SIZE;
113
+ const sycl::range<2> grid(n_seq, (n_head * head_dim + num_warps - 1) / num_warps);
114
+ const sycl::range<2> block(1, threads);
115
+ stream->parallel_for(
116
+ sycl::nd_range<2>(grid * block, block),
117
+ [=](sycl::nd_item<2> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
118
+ ssm_scan_f32_group<256 / WARP_SIZE, 256>(
119
+ src0, src1, src2, src3, src4, src5, src6, dst,
120
+ src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2, src3_nb1,
121
+ src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, head_dim, n_group, n_tok, K, item);
122
+ });
123
+ } else {
124
+ GGML_ABORT("ssm_scan: unsupported d_state (must be 128 or 256)");
125
+ }
126
+ }
127
+
128
+ inline void ggml_sycl_op_ssm_scan(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
129
+ const ggml_tensor * src0 = dst->src[0];
130
+ const ggml_tensor * src1 = dst->src[1];
131
+ const ggml_tensor * src2 = dst->src[2];
132
+ const ggml_tensor * src3 = dst->src[3];
133
+ const ggml_tensor * src4 = dst->src[4];
134
+ const ggml_tensor * src5 = dst->src[5];
135
+ const ggml_tensor * src6 = dst->src[6];
136
+
137
+ GGML_ASSERT(src0->type == GGML_TYPE_F32);
138
+ GGML_ASSERT(src6->type == GGML_TYPE_I32);
139
+ GGML_ASSERT(dst->type == GGML_TYPE_F32);
140
+
141
+ const int64_t nc = src0->ne[0];
142
+ const int64_t nr = src0->ne[1];
143
+ const int64_t nh = src1->ne[1];
144
+ const int64_t ng = src4->ne[1];
145
+ const int64_t n_t = src1->ne[2];
146
+ const int64_t n_s = src1->ne[3];
147
+ const int64_t K = ggml_get_op_params_i32(dst, 0);
148
+ const int64_t s_off = ggml_nelements(src1) * sizeof(float);
149
+
150
+ GGML_ASSERT(K >= 1);
151
+ GGML_ASSERT(ggml_nelements(src1) + K * nc * nr * nh * n_s == ggml_nelements(dst));
152
+ GGML_ASSERT(src3->ne[0] == 1 || K == 1);
153
+
154
+ dpct::queue_ptr stream = ctx.stream();
155
+ SYCL_CHECK(ggml_sycl_set_device(ctx.device));
156
+
157
+ ssm_scan_f32_sycl(
158
+ static_cast<const float *>(src0->data), static_cast<const float *>(src1->data),
159
+ static_cast<const float *>(src2->data), static_cast<const float *>(src3->data),
160
+ static_cast<const float *>(src4->data), static_cast<const float *>(src5->data),
161
+ static_cast<const int32_t *>(src6->data), static_cast<float *>(dst->data),
162
+ src0->nb[2], src0->nb[3], src1->nb[2], src1->nb[3], src2->nb[1], src2->nb[2],
163
+ src3->nb[1], src4->nb[2], src4->nb[3], src5->nb[2], src5->nb[3],
164
+ s_off, nc, nr, nh, ng, n_t, n_s, K, stream);
165
+ }
166
+
167
+ void ggml_sycl_ssm_scan(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
168
+ scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/7);
169
+ ggml_sycl_op_ssm_scan(ctx, dst);
170
+ }
ggml/src/ggml-sycl/ssm_scan.hpp ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ #pragma once
2
+
3
+ #include "common.hpp"
4
+
5
+ void ggml_sycl_ssm_scan(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
ggml/src/ggml-sycl/sycl_hw.cpp ADDED
@@ -0,0 +1,67 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #include "sycl_hw.hpp"
2
+
3
+ using namespace std;
4
+
5
+ /*defined in
6
+ * /opt/intel/oneapi/compiler/latest/include/sycl/ext/oneapi/experimental/device_architecture.def
7
+ */
8
+ static map<gpu_arch, std::pair<const char*, sycl_intel_gpu_family>> arch2name = {
9
+ {gpu_arch::intel_gpu_bdw, {"intel_gpu_bdw", GPU_FAMILY_IGPU_NON_XE}},
10
+ {gpu_arch::intel_gpu_skl, {"intel_gpu_skl", GPU_FAMILY_IGPU_NON_XE}},
11
+ {gpu_arch::intel_gpu_kbl, {"intel_gpu_kbl", GPU_FAMILY_IGPU_NON_XE}},
12
+ {gpu_arch::intel_gpu_cfl, {"intel_gpu_cfl", GPU_FAMILY_IGPU_NON_XE}},
13
+ {gpu_arch::intel_gpu_apl, {"intel_gpu_apl", GPU_FAMILY_IGPU_NON_XE}},
14
+ {gpu_arch::intel_gpu_glk, {"intel_gpu_glk", GPU_FAMILY_IGPU_NON_XE}},
15
+ {gpu_arch::intel_gpu_whl, {"intel_gpu_whl", GPU_FAMILY_IGPU_NON_XE}},
16
+ {gpu_arch::intel_gpu_aml, {"intel_gpu_aml", GPU_FAMILY_IGPU_NON_XE}},
17
+ {gpu_arch::intel_gpu_cml, {"intel_gpu_cml", GPU_FAMILY_IGPU_NON_XE}},
18
+ {gpu_arch::intel_gpu_icllp, {"intel_gpu_icllp", GPU_FAMILY_IGPU_NON_XE}},
19
+ {gpu_arch::intel_gpu_ehl, {"intel_gpu_ehl", GPU_FAMILY_IGPU_NON_XE}},
20
+ {gpu_arch::intel_gpu_tgllp, {"intel_gpu_tgllp", GPU_FAMILY_IGPU_NON_XE}},
21
+ {gpu_arch::intel_gpu_rkl, {"intel_gpu_rkl", GPU_FAMILY_IGPU_NON_XE}},
22
+ {gpu_arch::intel_gpu_adl_s, {"intel_gpu_adl_s", GPU_FAMILY_IGPU_NON_XE}},
23
+ {gpu_arch::intel_gpu_adl_p, {"intel_gpu_adl_p", GPU_FAMILY_IGPU_NON_XE}},
24
+ {gpu_arch::intel_gpu_adl_n, {"intel_gpu_adl_n", GPU_FAMILY_IGPU_NON_XE}},
25
+ {gpu_arch::intel_gpu_dg1, {"intel_gpu_dg1", GPU_FAMILY_DGPU_CLIENT_GAME}},
26
+ {gpu_arch::intel_gpu_acm_g10, {"intel_gpu_acm_g10", GPU_FAMILY_DGPU_CLIENT_GAME}},
27
+ {gpu_arch::intel_gpu_acm_g11, {"intel_gpu_acm_g11", GPU_FAMILY_DGPU_CLIENT_GAME}},
28
+ {gpu_arch::intel_gpu_acm_g12, {"intel_gpu_acm_g12", GPU_FAMILY_DGPU_CLIENT_GAME}},
29
+ {gpu_arch::intel_gpu_pvc, {"intel_gpu_pvc", GPU_FAMILY_DGPU_CLOUD}},
30
+ {gpu_arch::intel_gpu_pvc_vg, {"intel_gpu_pvc_vg", GPU_FAMILY_DGPU_CLOUD}},
31
+ {gpu_arch::intel_gpu_mtl_u, {"intel_gpu_mtl_u", GPU_FAMILY_IGPU_XE}},
32
+ {gpu_arch::intel_gpu_mtl_h, {"intel_gpu_mtl_h", GPU_FAMILY_IGPU_XE}},
33
+ {gpu_arch::intel_gpu_arl_h, {"intel_gpu_arl_h", GPU_FAMILY_IGPU_XE}},
34
+ {gpu_arch::intel_gpu_bmg_g21, {"intel_gpu_bmg_g21", GPU_FAMILY_DGPU_CLIENT_GAME}},
35
+ {gpu_arch::intel_gpu_bmg_g31, {"intel_gpu_bmg_g31", GPU_FAMILY_DGPU_CLIENT_GAME}},
36
+ {gpu_arch::intel_gpu_lnl_m, {"intel_gpu_lnl_m", GPU_FAMILY_IGPU_XE}},
37
+ {gpu_arch::intel_gpu_ptl_h, {"intel_gpu_ptl_h", GPU_FAMILY_IGPU_XE}},
38
+ {gpu_arch::intel_gpu_ptl_u, {"intel_gpu_ptl_u", GPU_FAMILY_IGPU_XE}},
39
+ {gpu_arch::intel_gpu_wcl, {"intel_gpu_wcl", GPU_FAMILY_IGPU_XE}}
40
+ };
41
+
42
+
43
+ sycl_hw_info get_device_hw_info(sycl::device* device_ptr) {
44
+ sycl_hw_info res;
45
+ int32_t id =
46
+ device_ptr->get_info<sycl::ext::intel::info::device::device_id>();
47
+ res.device_id = id;
48
+
49
+ res.name = device_ptr->get_info<sycl::info::device::name>();
50
+
51
+ syclex::architecture arch =
52
+ device_ptr->get_info<syclex::info::device::architecture>();
53
+ res.arch = arch;
54
+
55
+ map<syclex::architecture,
56
+ std::pair<const char*, sycl_intel_gpu_family>>::iterator it =
57
+ arch2name.find(res.arch);
58
+ if (it != arch2name.end()) {
59
+ res.arch_name = it->second.first;
60
+ res.gpu_family = it->second.second;
61
+ } else {
62
+ res.arch_name = "unknown";
63
+ res.gpu_family = GPU_FAMILY_UKNOWN;
64
+ }
65
+
66
+ return res;
67
+ }
ggml/src/ggml-sycl/sycl_hw.hpp ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #ifndef SYCL_HW_HPP
2
+ #define SYCL_HW_HPP
3
+
4
+ #include <algorithm>
5
+ #include <stdio.h>
6
+ #include <vector>
7
+ #include <map>
8
+
9
+ #include <sycl/sycl.hpp>
10
+
11
+ namespace syclex = sycl::ext::oneapi::experimental;
12
+ using gpu_arch = sycl::ext::oneapi::experimental::architecture;
13
+
14
+ // It's used to mark the GPU computing capacity
15
+ // The value must flow the order of performance.
16
+ enum sycl_intel_gpu_family {
17
+ GPU_FAMILY_UKNOWN = -1,
18
+ // iGPU without Xe core, before Meteor Lake iGPU(Xe)
19
+ GPU_FAMILY_IGPU_NON_XE = 0,
20
+ // iGPU with Xe core, Meteor Lake iGPU or newer.
21
+ GPU_FAMILY_IGPU_XE = 1,
22
+ // dGPU for gaming in client/data center (DG1/FLex 140 or newer).
23
+ GPU_FAMILY_DGPU_CLIENT_GAME = 2,
24
+ // dGPU for AI in cloud, PVC or newer.
25
+ GPU_FAMILY_DGPU_CLOUD = 3
26
+ };
27
+
28
+ struct sycl_hw_info {
29
+ syclex::architecture arch;
30
+ const char* arch_name;
31
+ int32_t device_id;
32
+ std::string name;
33
+ sycl_intel_gpu_family gpu_family;
34
+ };
35
+
36
+ sycl_hw_info get_device_hw_info(sycl::device *device_ptr);
37
+
38
+ #endif // SYCL_HW_HPP
ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq112-dv112.cpp ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ // This file has been autogenerated by generate_cu_files.py, do not edit manually.
2
+
3
+ #include "../fattn-tile.hpp"
4
+
5
+ DECL_FATTN_TILE_CASE(112, 112);
ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq128-dv128.cpp ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ // This file has been autogenerated by generate_cu_files.py, do not edit manually.
2
+
3
+ #include "../fattn-tile.hpp"
4
+
5
+ DECL_FATTN_TILE_CASE(128, 128);
ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq256-dv256.cpp ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ // This file has been autogenerated by generate_cu_files.py, do not edit manually.
2
+
3
+ #include "../fattn-tile.hpp"
4
+
5
+ DECL_FATTN_TILE_CASE(256, 256);
ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq40-dv40.cpp ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ // This file has been autogenerated by generate_cu_files.py, do not edit manually.
2
+
3
+ #include "../fattn-tile.hpp"
4
+
5
+ DECL_FATTN_TILE_CASE(40, 40);
ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq512-dv512.cpp ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ // This file has been autogenerated by generate_cu_files.py, do not edit manually.
2
+
3
+ #include "../fattn-tile.hpp"
4
+
5
+ DECL_FATTN_TILE_CASE(512, 512);
6
+
ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq576-dv512.cpp ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ // This file has been autogenerated by generate_cu_files.py, do not edit manually.
2
+
3
+ #include "../fattn-tile.hpp"
4
+
5
+ DECL_FATTN_TILE_CASE(576, 512);
ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq64-dv64.cpp ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ // This file has been autogenerated by generate_cu_files.py, do not edit manually.
2
+
3
+ #include "../fattn-tile.hpp"
4
+
5
+ DECL_FATTN_TILE_CASE(64, 64);
ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq72-dv72.cpp ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ // This file has been autogenerated by generate_cu_files.py, do not edit manually.
2
+
3
+ #include "../fattn-tile.hpp"
4
+
5
+ DECL_FATTN_TILE_CASE(72, 72);
ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq80-dv80.cpp ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ // This file has been autogenerated by generate_cu_files.py, do not edit manually.
2
+
3
+ #include "../fattn-tile.hpp"
4
+
5
+ DECL_FATTN_TILE_CASE(80, 80);
ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq96-dv96.cpp ADDED
@@ -0,0 +1,5 @@
 
 
 
 
 
 
1
+ // This file has been autogenerated by generate_cu_files.py, do not edit manually.
2
+
3
+ #include "../fattn-tile.hpp"
4
+
5
+ DECL_FATTN_TILE_CASE(96, 96);
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-f16.cpp ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ // This file has been autogenerated by generate_cu_files.py, do not edit manually.
2
+
3
+ #include "../fattn-vec.hpp"
4
+
5
+ DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F16, GGML_TYPE_F16);
6
+ DECL_FATTN_VEC_CASE(128, GGML_TYPE_F16, GGML_TYPE_F16);
7
+ DECL_FATTN_VEC_CASE(256, GGML_TYPE_F16, GGML_TYPE_F16);
8
+ DECL_FATTN_VEC_CASE(512, GGML_TYPE_F16, GGML_TYPE_F16);
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_0.cpp ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ // This file has been autogenerated by generate_cu_files.py, do not edit manually.
2
+
3
+ #include "../fattn-vec.hpp"
4
+
5
+ DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q4_0);
6
+ DECL_FATTN_VEC_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q4_0);
7
+ DECL_FATTN_VEC_CASE(256, GGML_TYPE_F16, GGML_TYPE_Q4_0);
8
+ DECL_FATTN_VEC_CASE(512, GGML_TYPE_F16, GGML_TYPE_Q4_0);
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_1.cpp ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ // This file has been autogenerated by generate_cu_files.py, do not edit manually.
2
+
3
+ #include "../fattn-vec.hpp"
4
+
5
+ DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q4_1);
6
+ DECL_FATTN_VEC_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q4_1);
7
+ DECL_FATTN_VEC_CASE(256, GGML_TYPE_F16, GGML_TYPE_Q4_1);
8
+ DECL_FATTN_VEC_CASE(512, GGML_TYPE_F16, GGML_TYPE_Q4_1);
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_0.cpp ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ // This file has been autogenerated by generate_cu_files.py, do not edit manually.
2
+
3
+ #include "../fattn-vec.hpp"
4
+
5
+ DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q5_0);
6
+ DECL_FATTN_VEC_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q5_0);
7
+ DECL_FATTN_VEC_CASE(256, GGML_TYPE_F16, GGML_TYPE_Q5_0);
8
+ DECL_FATTN_VEC_CASE(512, GGML_TYPE_F16, GGML_TYPE_Q5_0);
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_1.cpp ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ // This file has been autogenerated by generate_cu_files.py, do not edit manually.
2
+
3
+ #include "../fattn-vec.hpp"
4
+
5
+ DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q5_1);
6
+ DECL_FATTN_VEC_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q5_1);
7
+ DECL_FATTN_VEC_CASE(256, GGML_TYPE_F16, GGML_TYPE_Q5_1);
8
+ DECL_FATTN_VEC_CASE(512, GGML_TYPE_F16, GGML_TYPE_Q5_1);
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q8_0.cpp ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ // This file has been autogenerated by generate_cu_files.py, do not edit manually.
2
+
3
+ #include "../fattn-vec.hpp"
4
+
5
+ DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q8_0);
6
+ DECL_FATTN_VEC_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q8_0);
7
+ DECL_FATTN_VEC_CASE(256, GGML_TYPE_F16, GGML_TYPE_Q8_0);
8
+ DECL_FATTN_VEC_CASE(512, GGML_TYPE_F16, GGML_TYPE_Q8_0);
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-f16.cpp ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ // This file has been autogenerated by generate_cu_files.py, do not edit manually.
2
+
3
+ #include "../fattn-vec.hpp"
4
+
5
+ DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_0, GGML_TYPE_F16);
6
+ DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_F16);
7
+ DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_0, GGML_TYPE_F16);
8
+ DECL_FATTN_VEC_CASE(512, GGML_TYPE_Q4_0, GGML_TYPE_F16);
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_0.cpp ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ // This file has been autogenerated by generate_cu_files.py, do not edit manually.
2
+
3
+ #include "../fattn-vec.hpp"
4
+
5
+ DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0);
6
+ DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0);
7
+ DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0);
8
+ DECL_FATTN_VEC_CASE(512, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0);
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_1.cpp ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ // This file has been autogenerated by generate_cu_files.py, do not edit manually.
2
+
3
+ #include "../fattn-vec.hpp"
4
+
5
+ DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1);
6
+ DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1);
7
+ DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1);
8
+ DECL_FATTN_VEC_CASE(512, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1);
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_0.cpp ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ // This file has been autogenerated by generate_cu_files.py, do not edit manually.
2
+
3
+ #include "../fattn-vec.hpp"
4
+
5
+ DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_0, GGML_TYPE_Q5_0);
6
+ DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q5_0);
7
+ DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_0, GGML_TYPE_Q5_0);
8
+ DECL_FATTN_VEC_CASE(512, GGML_TYPE_Q4_0, GGML_TYPE_Q5_0);
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_1.cpp ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ // This file has been autogenerated by generate_cu_files.py, do not edit manually.
2
+
3
+ #include "../fattn-vec.hpp"
4
+
5
+ DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_0, GGML_TYPE_Q5_1);
6
+ DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q5_1);
7
+ DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_0, GGML_TYPE_Q5_1);
8
+ DECL_FATTN_VEC_CASE(512, GGML_TYPE_Q4_0, GGML_TYPE_Q5_1);
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q8_0.cpp ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ // This file has been autogenerated by generate_cu_files.py, do not edit manually.
2
+
3
+ #include "../fattn-vec.hpp"
4
+
5
+ DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0);
6
+ DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0);
7
+ DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0);
8
+ DECL_FATTN_VEC_CASE(512, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0);
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-f16.cpp ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ // This file has been autogenerated by generate_cu_files.py, do not edit manually.
2
+
3
+ #include "../fattn-vec.hpp"
4
+
5
+ DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_1, GGML_TYPE_F16);
6
+ DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_F16);
7
+ DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_1, GGML_TYPE_F16);
8
+ DECL_FATTN_VEC_CASE(512, GGML_TYPE_Q4_1, GGML_TYPE_F16);
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_0.cpp ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ // This file has been autogenerated by generate_cu_files.py, do not edit manually.
2
+
3
+ #include "../fattn-vec.hpp"
4
+
5
+ DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0);
6
+ DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0);
7
+ DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0);
8
+ DECL_FATTN_VEC_CASE(512, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0);
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_1.cpp ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ // This file has been autogenerated by generate_cu_files.py, do not edit manually.
2
+
3
+ #include "../fattn-vec.hpp"
4
+
5
+ DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1);
6
+ DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1);
7
+ DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1);
8
+ DECL_FATTN_VEC_CASE(512, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1);
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_0.cpp ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ // This file has been autogenerated by generate_cu_files.py, do not edit manually.
2
+
3
+ #include "../fattn-vec.hpp"
4
+
5
+ DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0);
6
+ DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0);
7
+ DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0);
8
+ DECL_FATTN_VEC_CASE(512, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0);
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_1.cpp ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ // This file has been autogenerated by generate_cu_files.py, do not edit manually.
2
+
3
+ #include "../fattn-vec.hpp"
4
+
5
+ DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_1, GGML_TYPE_Q5_1);
6
+ DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q5_1);
7
+ DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_1, GGML_TYPE_Q5_1);
8
+ DECL_FATTN_VEC_CASE(512, GGML_TYPE_Q4_1, GGML_TYPE_Q5_1);
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q8_0.cpp ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ // This file has been autogenerated by generate_cu_files.py, do not edit manually.
2
+
3
+ #include "../fattn-vec.hpp"
4
+
5
+ DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_1, GGML_TYPE_Q8_0);
6
+ DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q8_0);
7
+ DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_1, GGML_TYPE_Q8_0);
8
+ DECL_FATTN_VEC_CASE(512, GGML_TYPE_Q4_1, GGML_TYPE_Q8_0);
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-f16.cpp ADDED
@@ -0,0 +1,8 @@
 
 
 
 
 
 
 
 
 
1
+ // This file has been autogenerated by generate_cu_files.py, do not edit manually.
2
+
3
+ #include "../fattn-vec.hpp"
4
+
5
+ DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q5_0, GGML_TYPE_F16);
6
+ DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_F16);
7
+ DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q5_0, GGML_TYPE_F16);
8
+ DECL_FATTN_VEC_CASE(512, GGML_TYPE_Q5_0, GGML_TYPE_F16);