Parakon runtime: llama.cpp fork with fused 1-bit CUDA/Metal/CPU kernels (part 6)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- ggml/src/ggml-sycl/quants.hpp +204 -0
- ggml/src/ggml-sycl/repeat_back.cpp +76 -0
- ggml/src/ggml-sycl/repeat_back.hpp +8 -0
- ggml/src/ggml-sycl/roll.cpp +122 -0
- ggml/src/ggml-sycl/roll.hpp +20 -0
- ggml/src/ggml-sycl/rope.cpp +651 -0
- ggml/src/ggml-sycl/rope.hpp +26 -0
- ggml/src/ggml-sycl/set.cpp +73 -0
- ggml/src/ggml-sycl/set.hpp +5 -0
- ggml/src/ggml-sycl/set_rows.cpp +577 -0
- ggml/src/ggml-sycl/set_rows.hpp +8 -0
- ggml/src/ggml-sycl/softmax.cpp +425 -0
- ggml/src/ggml-sycl/softmax.hpp +24 -0
- ggml/src/ggml-sycl/solve_tri.cpp +172 -0
- ggml/src/ggml-sycl/solve_tri.hpp +8 -0
- ggml/src/ggml-sycl/ssm_conv.cpp +136 -0
- ggml/src/ggml-sycl/ssm_conv.hpp +5 -0
- ggml/src/ggml-sycl/ssm_scan.cpp +170 -0
- ggml/src/ggml-sycl/ssm_scan.hpp +5 -0
- ggml/src/ggml-sycl/sycl_hw.cpp +67 -0
- ggml/src/ggml-sycl/sycl_hw.hpp +38 -0
- ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq112-dv112.cpp +5 -0
- ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq128-dv128.cpp +5 -0
- ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq256-dv256.cpp +5 -0
- ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq40-dv40.cpp +5 -0
- ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq512-dv512.cpp +6 -0
- ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq576-dv512.cpp +5 -0
- ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq64-dv64.cpp +5 -0
- ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq72-dv72.cpp +5 -0
- ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq80-dv80.cpp +5 -0
- ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq96-dv96.cpp +5 -0
- ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-f16.cpp +8 -0
- ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_0.cpp +8 -0
- ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_1.cpp +8 -0
- ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_0.cpp +8 -0
- ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_1.cpp +8 -0
- ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q8_0.cpp +8 -0
- ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-f16.cpp +8 -0
- ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_0.cpp +8 -0
- ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_1.cpp +8 -0
- ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_0.cpp +8 -0
- ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_1.cpp +8 -0
- ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q8_0.cpp +8 -0
- ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-f16.cpp +8 -0
- ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_0.cpp +8 -0
- ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_1.cpp +8 -0
- ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_0.cpp +8 -0
- ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_1.cpp +8 -0
- ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q8_0.cpp +8 -0
- ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-f16.cpp +8 -0
ggml/src/ggml-sycl/quants.hpp
ADDED
|
@@ -0,0 +1,204 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//
|
| 2 |
+
// MIT license
|
| 3 |
+
// Copyright (C) 2025 Codeplay Software Ltd.
|
| 4 |
+
// Copyright (C) 2025 Intel Corporation
|
| 5 |
+
// SPDX-License-Identifier: MIT
|
| 6 |
+
//
|
| 7 |
+
|
| 8 |
+
//
|
| 9 |
+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
| 10 |
+
// See https://llvm.org/LICENSE.txt for license information.
|
| 11 |
+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
| 12 |
+
//
|
| 13 |
+
|
| 14 |
+
#ifndef GGML_SYCL_QUANTS_HPP
|
| 15 |
+
#define GGML_SYCL_QUANTS_HPP
|
| 16 |
+
|
| 17 |
+
#include <utility>
|
| 18 |
+
|
| 19 |
+
#include "ggml-common.h"
|
| 20 |
+
#include "ggml.h"
|
| 21 |
+
|
| 22 |
+
namespace ggml_sycl_reordered {
|
| 23 |
+
|
| 24 |
+
// The reordered block moves quants (qs) and scales(d) to two
|
| 25 |
+
// uniform regions of memory that is contiguous in the same tensor.
|
| 26 |
+
// What this means is that instead of having:
|
| 27 |
+
// [d0, qs0] [d1, qs1] [d2, qs2] ... [dN, qsN]
|
| 28 |
+
// We have:
|
| 29 |
+
// [qs0, qs1, qs2, ..., qsN] [d0, d1, d2, ..., dN]
|
| 30 |
+
//
|
| 31 |
+
// Notes: out-of-bounds qs will run into d values
|
| 32 |
+
// Alignment relies on the allocated size of qs
|
| 33 |
+
|
| 34 |
+
template <ggml_type type> struct block_q_t;
|
| 35 |
+
|
| 36 |
+
// qk number of weights / quants in a block
|
| 37 |
+
// qr number of weights in a byte (described as 'before dequantization')
|
| 38 |
+
// for quantization types that has low and high bits split, qr is calculated with
|
| 39 |
+
// using the lower bits, e.g for Q6 quants QR6 is 2
|
| 40 |
+
// qi number of 32 bit integers needed to represent all the quants from a block (`qs` field)
|
| 41 |
+
// See ggml-common.h to see how these are calculated
|
| 42 |
+
template <> struct block_q_t<GGML_TYPE_Q4_0> {
|
| 43 |
+
struct traits {
|
| 44 |
+
static constexpr uint32_t qk = QK4_0;
|
| 45 |
+
static constexpr uint32_t qi = QI4_0;
|
| 46 |
+
static constexpr uint32_t qr = QR4_0;
|
| 47 |
+
static constexpr uint32_t vdr_mmvq = 2;
|
| 48 |
+
};
|
| 49 |
+
|
| 50 |
+
static constexpr std::pair<int, int> get_block_offset(const int block_index, const int /* nblocks */) {
|
| 51 |
+
return { block_index * (QK4_0 / QR4_0), 0 };
|
| 52 |
+
}
|
| 53 |
+
|
| 54 |
+
static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) {
|
| 55 |
+
return { (ncols / QR4_0 * nrows) + block_index * sizeof(ggml_half), 0 };
|
| 56 |
+
}
|
| 57 |
+
|
| 58 |
+
static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }
|
| 59 |
+
};
|
| 60 |
+
|
| 61 |
+
template <> struct block_q_t<GGML_TYPE_Q2_K> {
|
| 62 |
+
struct traits {
|
| 63 |
+
static constexpr uint32_t qk = QK_K;
|
| 64 |
+
static constexpr uint32_t qi = QI2_K;
|
| 65 |
+
static constexpr uint32_t qr = QR2_K;
|
| 66 |
+
static constexpr uint32_t vdr_mmvq = 1;
|
| 67 |
+
};
|
| 68 |
+
|
| 69 |
+
// Reordered layout: [qs (QK_K/4 per block)] [scales (QK_K/16 per block)] [dm]
|
| 70 |
+
static constexpr std::pair<int, int> get_block_offset(const int block_index, const int /* n_blocks */) {
|
| 71 |
+
return { block_index * (QK_K / 4), 0 };
|
| 72 |
+
}
|
| 73 |
+
|
| 74 |
+
static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) {
|
| 75 |
+
auto nblocks = (nrows * (ncols / QK_K));
|
| 76 |
+
auto total_qs_bytes = nblocks * (QK_K / 4);
|
| 77 |
+
return { total_qs_bytes + block_index * (QK_K / 16),
|
| 78 |
+
total_qs_bytes + nblocks * (QK_K / 16) + block_index * sizeof(ggml_half2) };
|
| 79 |
+
}
|
| 80 |
+
|
| 81 |
+
static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }
|
| 82 |
+
};
|
| 83 |
+
|
| 84 |
+
template <> struct block_q_t<GGML_TYPE_Q3_K> {
|
| 85 |
+
struct traits {
|
| 86 |
+
static constexpr uint32_t qk = QK_K;
|
| 87 |
+
static constexpr uint32_t qi = QI3_K;
|
| 88 |
+
static constexpr uint32_t qr = QR3_K;
|
| 89 |
+
static constexpr uint32_t vdr_mmvq = 1;
|
| 90 |
+
};
|
| 91 |
+
|
| 92 |
+
// Reordered layout: [qs (QK_K/4 per block)] [hmask (QK_K/8 per block)] [scales] [d]
|
| 93 |
+
static constexpr std::pair<int, int> get_block_offset(const int block_index, const int n_blocks) {
|
| 94 |
+
auto qs_offset = block_index * (QK_K / 4);
|
| 95 |
+
auto hmask_offset = n_blocks * (QK_K / 4) + block_index * (QK_K / 8);
|
| 96 |
+
return { qs_offset, hmask_offset };
|
| 97 |
+
}
|
| 98 |
+
|
| 99 |
+
static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) {
|
| 100 |
+
auto nblocks = (nrows * (ncols / QK_K));
|
| 101 |
+
auto total_qs_bytes = nblocks * (QK_K / 4) + nblocks * (QK_K / 8);
|
| 102 |
+
return { total_qs_bytes + block_index * 12,
|
| 103 |
+
total_qs_bytes + nblocks * 12 + block_index * sizeof(ggml_half) };
|
| 104 |
+
}
|
| 105 |
+
|
| 106 |
+
static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }
|
| 107 |
+
};
|
| 108 |
+
|
| 109 |
+
template <> struct block_q_t<GGML_TYPE_Q4_K> {
|
| 110 |
+
struct traits {
|
| 111 |
+
static constexpr uint32_t qk = QK_K;
|
| 112 |
+
static constexpr uint32_t qi = QI4_K;
|
| 113 |
+
static constexpr uint32_t qr = QR4_K;
|
| 114 |
+
static constexpr uint32_t vdr_mmvq = 2;
|
| 115 |
+
};
|
| 116 |
+
|
| 117 |
+
static constexpr std::pair<int, int> get_block_offset(const int block_index, const int /* nblocks */) {
|
| 118 |
+
return { block_index * (traits::qk / traits::qr), 0 };
|
| 119 |
+
}
|
| 120 |
+
|
| 121 |
+
static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) {
|
| 122 |
+
auto nblocks = (nrows * (ncols / QK_K));
|
| 123 |
+
return { nblocks * (QK_K / 2) + (block_index * K_SCALE_SIZE),
|
| 124 |
+
(nblocks * QK_K / 2) + (nblocks * K_SCALE_SIZE) + (block_index * sizeof(ggml_half2)) };
|
| 125 |
+
}
|
| 126 |
+
|
| 127 |
+
static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }
|
| 128 |
+
};
|
| 129 |
+
|
| 130 |
+
template <> struct block_q_t<GGML_TYPE_Q5_K> {
|
| 131 |
+
struct traits {
|
| 132 |
+
static constexpr uint32_t qk = QK_K;
|
| 133 |
+
static constexpr uint32_t qi = QI5_K;
|
| 134 |
+
static constexpr uint32_t qr = QR5_K;
|
| 135 |
+
static constexpr uint32_t vdr_mmvq = 2;
|
| 136 |
+
};
|
| 137 |
+
|
| 138 |
+
// Reordered layout: [qs (QK_K/2 per block)] [qh (QK_K/8 per block)] [scales] [dm]
|
| 139 |
+
static constexpr std::pair<int, int> get_block_offset(const int block_index, const int n_blocks) {
|
| 140 |
+
auto qs_offset = block_index * (QK_K / 2);
|
| 141 |
+
auto qh_offset = n_blocks * (QK_K / 2) + block_index * (QK_K / 8);
|
| 142 |
+
return { qs_offset, qh_offset };
|
| 143 |
+
}
|
| 144 |
+
|
| 145 |
+
static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) {
|
| 146 |
+
auto nblocks = (nrows * (ncols / QK_K));
|
| 147 |
+
auto total_qs_bytes = nblocks * (QK_K / 2) + nblocks * (QK_K / 8);
|
| 148 |
+
return { total_qs_bytes + block_index * K_SCALE_SIZE,
|
| 149 |
+
total_qs_bytes + nblocks * K_SCALE_SIZE + block_index * sizeof(ggml_half2) };
|
| 150 |
+
}
|
| 151 |
+
|
| 152 |
+
static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }
|
| 153 |
+
};
|
| 154 |
+
|
| 155 |
+
template <> struct block_q_t<GGML_TYPE_Q6_K> {
|
| 156 |
+
struct traits {
|
| 157 |
+
static constexpr uint32_t qk = QK_K;
|
| 158 |
+
static constexpr uint32_t qi = QI6_K;
|
| 159 |
+
static constexpr uint32_t qr = QR6_K;
|
| 160 |
+
static constexpr uint32_t vdr_mmvq = 1;
|
| 161 |
+
};
|
| 162 |
+
|
| 163 |
+
static constexpr std::pair<int, int> get_block_offset(const int block_index, const int n_blocks) {
|
| 164 |
+
auto low_bits_index = block_index * (QK_K / QR6_K);
|
| 165 |
+
// the index of high bits it's after all low bits
|
| 166 |
+
auto high_bits_index = n_blocks * (QK_K / 2) + (block_index * (QK_K / 4));
|
| 167 |
+
return { low_bits_index, high_bits_index };
|
| 168 |
+
}
|
| 169 |
+
|
| 170 |
+
static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) {
|
| 171 |
+
auto nblocks = (nrows * (ncols / QK_K));
|
| 172 |
+
auto total_qs_bytes = nblocks * (QK_K / 2) + nblocks * (QK_K / 4);
|
| 173 |
+
auto block_scales = total_qs_bytes + block_index * (QK_K / 16);
|
| 174 |
+
auto sb_scale = total_qs_bytes + nblocks * (QK_K / 16) + block_index * sizeof(ggml_half);
|
| 175 |
+
return { block_scales, sb_scale };
|
| 176 |
+
}
|
| 177 |
+
|
| 178 |
+
static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; }
|
| 179 |
+
};
|
| 180 |
+
|
| 181 |
+
template <> struct block_q_t<GGML_TYPE_Q8_0> {
|
| 182 |
+
struct traits {
|
| 183 |
+
static constexpr uint32_t qk = QK8_0; // 32
|
| 184 |
+
static constexpr uint32_t qi = QI8_0; // 8
|
| 185 |
+
static constexpr uint32_t qr = QR8_0; // 1
|
| 186 |
+
static constexpr uint32_t vdr_mmvq = 4;
|
| 187 |
+
};
|
| 188 |
+
|
| 189 |
+
// Q8_0 reorder layout: [qs0|qs1|...|qsN][d0|d1|...|dN]
|
| 190 |
+
// Each block has 32 int8 weights (32 bytes) followed by all scales
|
| 191 |
+
static constexpr std::pair<int, int> get_block_offset(const int block_index, const int /* nblocks */) {
|
| 192 |
+
return { block_index * QK8_0, 0 };
|
| 193 |
+
}
|
| 194 |
+
|
| 195 |
+
static constexpr std::pair<int, int> get_d_offset(int nrows, int ncols, const int block_index) {
|
| 196 |
+
return { (ncols * nrows) + block_index * sizeof(ggml_half), 0 };
|
| 197 |
+
}
|
| 198 |
+
|
| 199 |
+
static constexpr int block_to_q8_1_ratio() { return traits::qk / QK8_1; } // 1
|
| 200 |
+
};
|
| 201 |
+
|
| 202 |
+
} // namespace ggml_sycl_reordered
|
| 203 |
+
|
| 204 |
+
#endif // GGML_SYCL_QUANTS_HPP
|
ggml/src/ggml-sycl/repeat_back.cpp
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#include "repeat_back.hpp"
|
| 2 |
+
|
| 3 |
+
#include "common.hpp"
|
| 4 |
+
|
| 5 |
+
#define GGML_ASSERT_TENSOR_FITS_INT(t) \
|
| 6 |
+
GGML_ASSERT((t)->ne[0] < INT_MAX && (t)->ne[1] < INT_MAX && (t)->ne[2] < INT_MAX && (t)->ne[3] < INT_MAX)
|
| 7 |
+
|
| 8 |
+
void ggml_sycl_op_repeat_back(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
|
| 9 |
+
GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32);
|
| 10 |
+
GGML_ASSERT(dst->type == GGML_TYPE_F32);
|
| 11 |
+
|
| 12 |
+
const float * src0_dd = (const float *) dst->src[0]->data;
|
| 13 |
+
float * dst_dd = (float *) dst->data;
|
| 14 |
+
|
| 15 |
+
GGML_ASSERT_TENSOR_FITS_INT(dst);
|
| 16 |
+
GGML_ASSERT_TENSOR_FITS_INT(dst->src[0]);
|
| 17 |
+
|
| 18 |
+
const int ne0 = dst->ne[0], ne1 = dst->ne[1], ne2 = dst->ne[2], ne3 = dst->ne[3];
|
| 19 |
+
const int ne00 = dst->src[0]->ne[0], ne01 = dst->src[0]->ne[1], ne02 = dst->src[0]->ne[2],
|
| 20 |
+
ne03 = dst->src[0]->ne[3];
|
| 21 |
+
|
| 22 |
+
const int nr0 = ne00 / ne0;
|
| 23 |
+
const int nr1 = ne01 / ne1;
|
| 24 |
+
const int nr2 = ne02 / ne2;
|
| 25 |
+
const int nr3 = ne03 / ne3;
|
| 26 |
+
|
| 27 |
+
const int nb0 = dst->src[0]->nb[0];
|
| 28 |
+
const int nb1 = dst->src[0]->nb[1];
|
| 29 |
+
const int nb2 = dst->src[0]->nb[2];
|
| 30 |
+
const int nb3 = dst->src[0]->nb[3];
|
| 31 |
+
|
| 32 |
+
const char * base = (const char *) src0_dd;
|
| 33 |
+
|
| 34 |
+
const size_t total = (size_t) ne0 * ne1 * ne2 * ne3;
|
| 35 |
+
constexpr int BLOCK_SIZE = 256;
|
| 36 |
+
const int num_blocks = (total + BLOCK_SIZE - 1) / BLOCK_SIZE;
|
| 37 |
+
|
| 38 |
+
const float inv_ne0 = 1.0f / ne0;
|
| 39 |
+
const float inv_ne_01 = 1.0f / (ne0 * ne1);
|
| 40 |
+
const float inv_ne_012 = 1.0f / (ne0 * ne1 * ne2);
|
| 41 |
+
const int repeat_count = nr0 * nr1 * nr2 * nr3;
|
| 42 |
+
|
| 43 |
+
queue_ptr stream = ctx.stream();
|
| 44 |
+
|
| 45 |
+
stream->parallel_for(
|
| 46 |
+
sycl::nd_range<1>(sycl::range<1>(num_blocks * BLOCK_SIZE), sycl::range<1>(BLOCK_SIZE)),
|
| 47 |
+
[=](sycl::nd_item<1> item_ct1) {
|
| 48 |
+
const size_t i = item_ct1.get_global_linear_id();
|
| 49 |
+
if (i >= total) {
|
| 50 |
+
return;
|
| 51 |
+
}
|
| 52 |
+
|
| 53 |
+
const int i3 = (int) (i * inv_ne_012);
|
| 54 |
+
const int i2 = (int) (i * inv_ne_01) - i3 * ne2;
|
| 55 |
+
const int i1 = (int) (i * inv_ne0) - (int) (i * inv_ne_01) * ne1;
|
| 56 |
+
const int i0 = i - (int) (i * inv_ne0) * ne0;
|
| 57 |
+
|
| 58 |
+
int j0 = 0, j1 = 0, j2 = 0, j3 = 0;
|
| 59 |
+
float acc = 0.0f;
|
| 60 |
+
|
| 61 |
+
for (int j = 0; j < repeat_count; ++j) {
|
| 62 |
+
const float * ptr = (const float *) (base + (i0 + j0 * ne0) * nb0 + (i1 + j1 * ne1) * nb1 +
|
| 63 |
+
(i2 + j2 * ne2) * nb2 + (i3 + j3 * ne3) * nb3);
|
| 64 |
+
acc += *ptr;
|
| 65 |
+
|
| 66 |
+
int carry = (++j0 >= nr0);
|
| 67 |
+
j0 -= carry * nr0;
|
| 68 |
+
carry = (carry && (++j1 >= nr1));
|
| 69 |
+
j1 -= carry * nr1;
|
| 70 |
+
carry = (carry && (++j2 >= nr2));
|
| 71 |
+
j2 -= carry * nr2;
|
| 72 |
+
j3 += carry;
|
| 73 |
+
}
|
| 74 |
+
dst_dd[i] = acc;
|
| 75 |
+
});
|
| 76 |
+
}
|
ggml/src/ggml-sycl/repeat_back.hpp
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#ifndef GGML_SYCL_REPEAT_BACK_HPP
|
| 2 |
+
#define GGML_SYCL_REPEAT_BACK_HPP
|
| 3 |
+
|
| 4 |
+
#include "common.hpp"
|
| 5 |
+
|
| 6 |
+
void ggml_sycl_op_repeat_back(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
|
| 7 |
+
|
| 8 |
+
#endif // GGML_SYCL_REPEAT_BACK_HPP
|
ggml/src/ggml-sycl/roll.cpp
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#include "roll.hpp"
|
| 2 |
+
#include "common.hpp"
|
| 3 |
+
|
| 4 |
+
using namespace sycl;
|
| 5 |
+
|
| 6 |
+
static inline int wrap_add(int i, int shift, int n) {
|
| 7 |
+
|
| 8 |
+
int s = i + shift;
|
| 9 |
+
return (s >= n) ? (s - n) : s;
|
| 10 |
+
}
|
| 11 |
+
|
| 12 |
+
static void kernel_roll_fused_i0_i1(
|
| 13 |
+
queue &q,
|
| 14 |
+
const float *src_d,
|
| 15 |
+
float *dst_d,
|
| 16 |
+
int ne0, int ne1, int ne2, int ne3,
|
| 17 |
+
int sh0, int sh1, int sh2, int sh3)
|
| 18 |
+
{
|
| 19 |
+
if (ne0 == 0 || ne1 == 0 || ne2 == 0 || ne3 == 0) return;
|
| 20 |
+
|
| 21 |
+
|
| 22 |
+
const int stride1 = ne0;
|
| 23 |
+
const int stride2 = ne0 * ne1;
|
| 24 |
+
const int stride3 = ne0 * ne1 * ne2;
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
const int shNe0 = (ne0 - sh0) % ne0;
|
| 28 |
+
const int shNe1 = (ne1 - sh1) % ne1;
|
| 29 |
+
const int shNe2 = (ne2 - sh2) % ne2;
|
| 30 |
+
const int shNe3 = (ne3 - sh3) % ne3;
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
const size_t g0 = (size_t) ne3;
|
| 34 |
+
const size_t g1 = (size_t) ne2;
|
| 35 |
+
const size_t g2 = (size_t) (ne1 * ne0);
|
| 36 |
+
|
| 37 |
+
const range<3> global{ g0, g1, g2 };
|
| 38 |
+
|
| 39 |
+
q.submit([&](handler &h) {
|
| 40 |
+
h.parallel_for(global, [=](id<3> idx) {
|
| 41 |
+
const int i3 = (int) idx[0];
|
| 42 |
+
const int i2 = (int) idx[1];
|
| 43 |
+
|
| 44 |
+
const int fused = (int) idx[2];
|
| 45 |
+
const int i1 = fused / ne0;
|
| 46 |
+
const int i0 = fused - i1 * ne0; // fused % ne0
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
const int idx_dst = i0
|
| 50 |
+
+ i1 * stride1
|
| 51 |
+
+ i2 * stride2
|
| 52 |
+
+ i3 * stride3;
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
const int s0 = wrap_add(i0, shNe0, ne0);
|
| 56 |
+
const int s1 = wrap_add(i1, shNe1, ne1);
|
| 57 |
+
const int s2 = wrap_add(i2, shNe2, ne2);
|
| 58 |
+
const int s3 = wrap_add(i3, shNe3, ne3);
|
| 59 |
+
|
| 60 |
+
const int idx_src = s0
|
| 61 |
+
+ s1 * stride1
|
| 62 |
+
+ s2 * stride2
|
| 63 |
+
+ s3 * stride3;
|
| 64 |
+
|
| 65 |
+
dst_d[idx_dst] = src_d[idx_src];
|
| 66 |
+
});
|
| 67 |
+
});
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
void ggml_sycl_roll(ggml_backend_sycl_context & ctx, ggml_tensor *dst) {
|
| 71 |
+
GGML_ASSERT(dst->type == GGML_TYPE_F32);
|
| 72 |
+
|
| 73 |
+
const ggml_tensor *src = dst->src[0];
|
| 74 |
+
GGML_ASSERT(src && src->type == GGML_TYPE_F32);
|
| 75 |
+
|
| 76 |
+
const int ne0 = (int) dst->ne[0];
|
| 77 |
+
const int ne1 = (int) dst->ne[1];
|
| 78 |
+
const int ne2 = (int) dst->ne[2];
|
| 79 |
+
const int ne3 = (int) dst->ne[3];
|
| 80 |
+
|
| 81 |
+
const int32_t *params = (const int32_t *) dst->op_params;
|
| 82 |
+
int shift0 = params[0];
|
| 83 |
+
int shift1 = params[1];
|
| 84 |
+
int shift2 = params[2];
|
| 85 |
+
int shift3 = params[3];
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
if ((shift0 | shift1 | shift2 | shift3) == 0) {
|
| 89 |
+
const size_t nb = ggml_nbytes(src);
|
| 90 |
+
queue *q = ctx.stream();
|
| 91 |
+
SYCL_CHECK(CHECK_TRY_ERROR(q->memcpy(dst->data, src->data, nb)));
|
| 92 |
+
return;
|
| 93 |
+
}
|
| 94 |
+
|
| 95 |
+
auto norm = [](int sh, int n) -> int {
|
| 96 |
+
if (n <= 0) return 0;
|
| 97 |
+
sh %= n;
|
| 98 |
+
if (sh < 0) sh += n;
|
| 99 |
+
return sh;
|
| 100 |
+
};
|
| 101 |
+
shift0 = norm(shift0, ne0);
|
| 102 |
+
shift1 = norm(shift1, ne1);
|
| 103 |
+
shift2 = norm(shift2, ne2);
|
| 104 |
+
shift3 = norm(shift3, ne3);
|
| 105 |
+
|
| 106 |
+
try {
|
| 107 |
+
queue *q = ctx.stream();
|
| 108 |
+
|
| 109 |
+
const float *src_d = (const float *) src->data;
|
| 110 |
+
float *dst_d = (float *) dst->data;
|
| 111 |
+
GGML_ASSERT(src_d && dst_d);
|
| 112 |
+
|
| 113 |
+
kernel_roll_fused_i0_i1(
|
| 114 |
+
*q, src_d, dst_d,
|
| 115 |
+
ne0, ne1, ne2, ne3,
|
| 116 |
+
shift0, shift1, shift2, shift3
|
| 117 |
+
);
|
| 118 |
+
} catch (const std::exception &e) {
|
| 119 |
+
std::fprintf(stderr, "[SYCL-ROLL] ERROR: %s\n", e.what());
|
| 120 |
+
throw;
|
| 121 |
+
}
|
| 122 |
+
}
|
ggml/src/ggml-sycl/roll.hpp
ADDED
|
@@ -0,0 +1,20 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//
|
| 2 |
+
// MIT license
|
| 3 |
+
// Copyright (C) 2024 Intel Corporation
|
| 4 |
+
// SPDX-License-Identifier: MIT
|
| 5 |
+
//
|
| 6 |
+
|
| 7 |
+
//
|
| 8 |
+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
| 9 |
+
// See https://llvm.org/LICENSE.txt for license information.
|
| 10 |
+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
| 11 |
+
//
|
| 12 |
+
|
| 13 |
+
#ifndef GGML_SYCL_ROLL_HPP
|
| 14 |
+
#define GGML_SYCL_ROLL_HPP
|
| 15 |
+
|
| 16 |
+
#include "common.hpp"
|
| 17 |
+
|
| 18 |
+
void ggml_sycl_roll(ggml_backend_sycl_context & ctx, ggml_tensor *dst);
|
| 19 |
+
|
| 20 |
+
#endif // GGML_SYCL_ROLL_HPP
|
ggml/src/ggml-sycl/rope.cpp
ADDED
|
@@ -0,0 +1,651 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#include "rope.hpp"
|
| 2 |
+
#include "convert.hpp"
|
| 3 |
+
#include "ggml-sycl/common.hpp"
|
| 4 |
+
#include "ggml.h"
|
| 5 |
+
|
| 6 |
+
struct rope_corr_dims {
|
| 7 |
+
float v[2];
|
| 8 |
+
};
|
| 9 |
+
|
| 10 |
+
struct mrope_sections {
|
| 11 |
+
int v[4];
|
| 12 |
+
};
|
| 13 |
+
|
| 14 |
+
static float rope_yarn_ramp(const float low, const float high, const int i0) {
|
| 15 |
+
const float y = (i0 / 2 - low) / sycl::max(0.001f, high - low);
|
| 16 |
+
return 1.0f - sycl::min(1.0f, sycl::max(0.0f, y));
|
| 17 |
+
}
|
| 18 |
+
|
| 19 |
+
template <bool forward>
|
| 20 |
+
static void rope_yarn(const float theta_extrap, const float freq_scale,
|
| 21 |
+
const rope_corr_dims corr_dims, const int64_t i0,
|
| 22 |
+
const float ext_factor, float mscale, float &cos_theta,
|
| 23 |
+
float &sin_theta) {
|
| 24 |
+
float theta_interp = freq_scale * theta_extrap;
|
| 25 |
+
float theta = theta_interp;
|
| 26 |
+
if (ext_factor != 0.0f) {
|
| 27 |
+
float ramp_mix =
|
| 28 |
+
rope_yarn_ramp(corr_dims.v[0], corr_dims.v[1], i0) * ext_factor;
|
| 29 |
+
theta = theta_interp * (1 - ramp_mix) + theta_extrap * ramp_mix;
|
| 30 |
+
|
| 31 |
+
mscale *= 1.0f + 0.1f * sycl::log(1.0f / freq_scale);
|
| 32 |
+
}
|
| 33 |
+
cos_theta = sycl::cos(theta) * mscale;
|
| 34 |
+
sin_theta = sycl::sin(theta) * mscale;
|
| 35 |
+
if (!forward) {
|
| 36 |
+
sin_theta *= -1.0f;
|
| 37 |
+
}
|
| 38 |
+
}
|
| 39 |
+
|
| 40 |
+
template <bool forward, bool has_ff, typename T, typename D>
|
| 41 |
+
static void rope_norm(const T *x, D *dst, const int ne00, const int ne01,
|
| 42 |
+
const int ne02, const int s01, const int s02,
|
| 43 |
+
const int s03, const int s1, const int s2, const int s3,
|
| 44 |
+
const int n_dims, const int n_offs, const int32_t *pos,
|
| 45 |
+
const float freq_scale, const float ext_factor,
|
| 46 |
+
const float attn_factor, const rope_corr_dims corr_dims,
|
| 47 |
+
const float theta_scale, const float *freq_factors,
|
| 48 |
+
const int64_t *row_indices, const int set_rows_stride) {
|
| 49 |
+
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
|
| 50 |
+
const int i0 = 2 * (item_ct1.get_local_range(1) * item_ct1.get_group(1) +
|
| 51 |
+
item_ct1.get_local_id(1));
|
| 52 |
+
|
| 53 |
+
if (i0 >= ne00) {
|
| 54 |
+
return;
|
| 55 |
+
}
|
| 56 |
+
|
| 57 |
+
const int row_dst = item_ct1.get_local_range(2) * item_ct1.get_group(2) +
|
| 58 |
+
item_ct1.get_local_id(2);
|
| 59 |
+
|
| 60 |
+
const uint32_t i3 = row_dst / (ne01 * ne02);
|
| 61 |
+
const uint32_t i2 = (row_dst - i3 * ne01 * ne02) / ne01;
|
| 62 |
+
const uint32_t i1 = row_dst - i3 * ne01 * ne02 - i2 * ne01;
|
| 63 |
+
|
| 64 |
+
int idst = i0 + i1 * s1 + i2 * s2 + i3 * s3;
|
| 65 |
+
const int ix = i0 + i1 * s01 + i2 * s02 + i3 * s03;
|
| 66 |
+
|
| 67 |
+
if (set_rows_stride != 0) {
|
| 68 |
+
idst = i1 * s1 + i0;
|
| 69 |
+
idst += row_indices[i2] * set_rows_stride;
|
| 70 |
+
}
|
| 71 |
+
|
| 72 |
+
const auto &store_coaelsced = [&](float x0, float x1) {
|
| 73 |
+
if constexpr (std::is_same_v<float, D>) {
|
| 74 |
+
sycl::float2 v = sycl::float2(x0, x1);
|
| 75 |
+
ggml_sycl_memcpy_1<8>(dst + idst, &v);
|
| 76 |
+
} else if constexpr (std::is_same_v<sycl::half, D>) {
|
| 77 |
+
sycl::half2 v = sycl::half2(x0, x1);
|
| 78 |
+
ggml_sycl_memcpy_1<4>(dst + idst, &v);
|
| 79 |
+
}
|
| 80 |
+
};
|
| 81 |
+
if (i0 < n_offs || i0 >= n_offs + n_dims) {
|
| 82 |
+
store_coaelsced(x[ix + 0], x[ix + 1]);
|
| 83 |
+
return;
|
| 84 |
+
}
|
| 85 |
+
|
| 86 |
+
const int iw = i0 - n_offs; // relative idx
|
| 87 |
+
|
| 88 |
+
const float theta_base = pos[i2] * dpct::pow(theta_scale, iw / 2.0f);
|
| 89 |
+
|
| 90 |
+
const float freq_factor = has_ff ? freq_factors[iw / 2] : 1.0f;
|
| 91 |
+
|
| 92 |
+
float cos_theta;
|
| 93 |
+
float sin_theta;
|
| 94 |
+
|
| 95 |
+
rope_yarn<forward>(theta_base / freq_factor, freq_scale, corr_dims, iw,
|
| 96 |
+
ext_factor, attn_factor, cos_theta, sin_theta);
|
| 97 |
+
|
| 98 |
+
const float x0 = x[ix + 0];
|
| 99 |
+
const float x1 = x[ix + 1];
|
| 100 |
+
|
| 101 |
+
store_coaelsced(x0 * cos_theta - x1 * sin_theta,
|
| 102 |
+
x0 * sin_theta + x1 * cos_theta);
|
| 103 |
+
}
|
| 104 |
+
|
| 105 |
+
template <bool forward, bool has_ff, typename T, typename D>
|
| 106 |
+
static void rope_neox(const T *x, D *dst, const int ne00, const int ne01,
|
| 107 |
+
const int ne02, const int s01, const int s02,
|
| 108 |
+
const int s03, const int s1, const int s2, const int s3,
|
| 109 |
+
const int n_dims, const int n_offs, const int32_t *pos,
|
| 110 |
+
const float freq_scale, const float ext_factor,
|
| 111 |
+
const float attn_factor, const rope_corr_dims corr_dims,
|
| 112 |
+
const float theta_scale, const float *freq_factors,
|
| 113 |
+
const int64_t *row_indices, const int set_rows_stride) {
|
| 114 |
+
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
|
| 115 |
+
const int i0 = 2 * (item_ct1.get_local_range(1) * item_ct1.get_group(1) +
|
| 116 |
+
item_ct1.get_local_id(1));
|
| 117 |
+
|
| 118 |
+
if (i0 >= ne00) {
|
| 119 |
+
return;
|
| 120 |
+
}
|
| 121 |
+
|
| 122 |
+
const int row_dst = item_ct1.get_local_range(2) * item_ct1.get_group(2) +
|
| 123 |
+
item_ct1.get_local_id(2);
|
| 124 |
+
|
| 125 |
+
const uint32_t i3 = row_dst / (ne01 * ne02);
|
| 126 |
+
const uint32_t i2 = (row_dst - i3 * ne01 * ne02) / ne01;
|
| 127 |
+
const uint32_t i1 = row_dst - i3 * ne01 * ne02 - i2 * ne01;
|
| 128 |
+
|
| 129 |
+
int idst = i0 / 2 + i1 * s1 + i2 * s2 + i3 * s3;
|
| 130 |
+
const int ix = i0 / 2 + i1 * s01 + i2 * s02 + i3 * s03;
|
| 131 |
+
|
| 132 |
+
if (set_rows_stride != 0) {
|
| 133 |
+
idst = i1 * s1 + i0 / 2;
|
| 134 |
+
idst += row_indices[i2] * set_rows_stride;
|
| 135 |
+
}
|
| 136 |
+
|
| 137 |
+
if (i0 < n_offs || i0 >= n_offs + n_dims) {
|
| 138 |
+
dst[idst + i0 / 2 + 0] = ggml_sycl_cast<D>(x[ix + i0 / 2 + 0]);
|
| 139 |
+
dst[idst + i0 / 2 + 1] = ggml_sycl_cast<D>(x[ix + i0 / 2 + 1]);
|
| 140 |
+
|
| 141 |
+
return;
|
| 142 |
+
}
|
| 143 |
+
|
| 144 |
+
const int iw = i0 - n_offs; // relative idx
|
| 145 |
+
|
| 146 |
+
const float theta_base = pos[i2] * dpct::pow(theta_scale, iw / 2.0f);
|
| 147 |
+
|
| 148 |
+
const float freq_factor = has_ff ? freq_factors[iw / 2] : 1.0f;
|
| 149 |
+
|
| 150 |
+
float cos_theta;
|
| 151 |
+
float sin_theta;
|
| 152 |
+
|
| 153 |
+
rope_yarn<forward>(theta_base / freq_factor, freq_scale, corr_dims, iw,
|
| 154 |
+
ext_factor, attn_factor, cos_theta, sin_theta);
|
| 155 |
+
|
| 156 |
+
// idst/ix point at channel i0/2; the first channel of the rotated pair is n_offs + iw/2 = i0/2 + n_offs/2
|
| 157 |
+
const float x0 = x[ix + n_offs / 2 + 0];
|
| 158 |
+
const float x1 = x[ix + n_offs / 2 + n_dims / 2];
|
| 159 |
+
|
| 160 |
+
dst[idst + n_offs / 2 + 0] = ggml_sycl_cast<D>(x0 * cos_theta - x1 * sin_theta);
|
| 161 |
+
dst[idst + n_offs / 2 + n_dims / 2] = ggml_sycl_cast<D>(x0 * sin_theta + x1 * cos_theta);
|
| 162 |
+
}
|
| 163 |
+
|
| 164 |
+
template <bool forward, bool has_ff, typename T>
|
| 165 |
+
static void rope_multi(const T *x, T *dst, const int ne00, const int ne01,
|
| 166 |
+
const int ne02, const int s01, const int s02,
|
| 167 |
+
const int s03, const int s1, const int s2, const int s3,
|
| 168 |
+
const int n_dims, const int n_offs, const int32_t *pos,
|
| 169 |
+
const float freq_scale, const float ext_factor,
|
| 170 |
+
const float attn_factor, const rope_corr_dims corr_dims,
|
| 171 |
+
const float theta_scale, const float *freq_factors,
|
| 172 |
+
const mrope_sections sections, const bool is_imrope) {
|
| 173 |
+
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
|
| 174 |
+
const int i0 = 2 * (item_ct1.get_local_range(1) * item_ct1.get_group(1) +
|
| 175 |
+
item_ct1.get_local_id(1));
|
| 176 |
+
|
| 177 |
+
if (i0 >= ne00) {
|
| 178 |
+
return;
|
| 179 |
+
}
|
| 180 |
+
|
| 181 |
+
const int row_dst = item_ct1.get_local_range(2) * item_ct1.get_group(2) +
|
| 182 |
+
item_ct1.get_local_id(2);
|
| 183 |
+
|
| 184 |
+
const uint32_t i3 = row_dst / (ne01 * ne02);
|
| 185 |
+
const uint32_t i2 = (row_dst - i3 * ne01 * ne02) / ne01;
|
| 186 |
+
const uint32_t i1 = row_dst - i3 * ne01 * ne02 - i2 * ne01;
|
| 187 |
+
|
| 188 |
+
int idst = i0 / 2 + i1 * s1 + i2 * s2 + i3 * s3;
|
| 189 |
+
const int ix = i0 / 2 + i1 * s01 + i2 * s02 + i3 * s03;
|
| 190 |
+
|
| 191 |
+
if (i0 < n_offs || i0 >= n_offs + n_dims) {
|
| 192 |
+
dst[idst + i0 / 2 + 0] = x[ix + i0 / 2 + 0];
|
| 193 |
+
dst[idst + i0 / 2 + 1] = x[ix + i0 / 2 + 1];
|
| 194 |
+
|
| 195 |
+
return;
|
| 196 |
+
}
|
| 197 |
+
|
| 198 |
+
const int iw = i0 - n_offs; // relative idx
|
| 199 |
+
|
| 200 |
+
const int sect_dims =
|
| 201 |
+
sections.v[0] + sections.v[1] + sections.v[2] + sections.v[3];
|
| 202 |
+
const int sec_w = sections.v[1] + sections.v[0];
|
| 203 |
+
const int sector = (iw / 2) % sect_dims;
|
| 204 |
+
|
| 205 |
+
float theta_base = 0.0;
|
| 206 |
+
if (is_imrope) {
|
| 207 |
+
if (sector % 3 == 1 && sector < 3 * sections.v[1]) { // h
|
| 208 |
+
theta_base = pos[i2 + ne02 * 1] * dpct::pow(theta_scale, iw / 2.0f);
|
| 209 |
+
} else if (sector % 3 == 2 && sector < 3 * sections.v[2]) { // w
|
| 210 |
+
theta_base = pos[i2 + ne02 * 2] * dpct::pow(theta_scale, iw / 2.0f);
|
| 211 |
+
} else if (sector % 3 == 0 && sector < 3 * sections.v[0]) { // t
|
| 212 |
+
theta_base = pos[i2] * dpct::pow(theta_scale, iw / 2.0f);
|
| 213 |
+
} else {
|
| 214 |
+
theta_base = pos[i2 + ne02 * 3] * dpct::pow(theta_scale, iw / 2.0f);
|
| 215 |
+
}
|
| 216 |
+
} else {
|
| 217 |
+
if (sector < sections.v[0]) {
|
| 218 |
+
theta_base = pos[i2] * dpct::pow(theta_scale, iw / 2.0f);
|
| 219 |
+
} else if (sector >= sections.v[0] && sector < sec_w) {
|
| 220 |
+
theta_base = pos[i2 + ne02 * 1] * dpct::pow(theta_scale, iw / 2.0f);
|
| 221 |
+
} else if (sector >= sec_w && sector < sec_w + sections.v[2]) {
|
| 222 |
+
theta_base = pos[i2 + ne02 * 2] * dpct::pow(theta_scale, iw / 2.0f);
|
| 223 |
+
} else if (sector >= sec_w + sections.v[2]) {
|
| 224 |
+
theta_base = pos[i2 + ne02 * 3] * dpct::pow(theta_scale, iw / 2.0f);
|
| 225 |
+
}
|
| 226 |
+
}
|
| 227 |
+
|
| 228 |
+
const float freq_factor = has_ff ? freq_factors[iw / 2] : 1.0f;
|
| 229 |
+
|
| 230 |
+
float cos_theta;
|
| 231 |
+
float sin_theta;
|
| 232 |
+
|
| 233 |
+
rope_yarn<forward>(theta_base / freq_factor, freq_scale, corr_dims, iw,
|
| 234 |
+
ext_factor, attn_factor, cos_theta, sin_theta);
|
| 235 |
+
|
| 236 |
+
// idst/ix point at channel i0/2; the first channel of the rotated pair is n_offs + iw/2 = i0/2 + n_offs/2
|
| 237 |
+
const float x0 = x[ix + n_offs / 2 + 0];
|
| 238 |
+
const float x1 = x[ix + n_offs / 2 + n_dims / 2];
|
| 239 |
+
|
| 240 |
+
dst[idst + n_offs / 2 + 0] = x0 * cos_theta - x1 * sin_theta;
|
| 241 |
+
dst[idst + n_offs / 2 + n_dims / 2] = x0 * sin_theta + x1 * cos_theta;
|
| 242 |
+
}
|
| 243 |
+
|
| 244 |
+
template <bool forward, bool has_ff, typename T>
|
| 245 |
+
static void rope_vision(const T *x, T *dst, const int ne00, const int ne01,
|
| 246 |
+
const int ne02, const int s01, const int s02,
|
| 247 |
+
const int s03, const int s1, const int s2, const int s3,
|
| 248 |
+
const int n_dims, const int32_t *pos,
|
| 249 |
+
const float freq_scale, const float ext_factor,
|
| 250 |
+
const float attn_factor, const rope_corr_dims corr_dims,
|
| 251 |
+
const float theta_scale, const float *freq_factors,
|
| 252 |
+
const mrope_sections sections) {
|
| 253 |
+
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
|
| 254 |
+
const int i0 = 2 * (item_ct1.get_local_range(1) * item_ct1.get_group(1) +
|
| 255 |
+
item_ct1.get_local_id(1));
|
| 256 |
+
|
| 257 |
+
if (i0 >= ne00) {
|
| 258 |
+
return;
|
| 259 |
+
}
|
| 260 |
+
|
| 261 |
+
const int row_dst = item_ct1.get_local_range(2) * item_ct1.get_group(2) +
|
| 262 |
+
item_ct1.get_local_id(2);
|
| 263 |
+
|
| 264 |
+
const uint32_t i3 = row_dst / (ne01 * ne02);
|
| 265 |
+
const uint32_t i2 = (row_dst - i3 * ne01 * ne02) / ne01;
|
| 266 |
+
const uint32_t i1 = row_dst - i3 * ne01 * ne02 - i2 * ne01;
|
| 267 |
+
|
| 268 |
+
int idst = i0 / 2 + i1 * s1 + i2 * s2 + i3 * s3;
|
| 269 |
+
const int ix = i0 / 2 + i1 * s01 + i2 * s02 + i3 * s03;
|
| 270 |
+
|
| 271 |
+
const int sect_dims = sections.v[0] + sections.v[1];
|
| 272 |
+
const int sec_w = sections.v[1] + sections.v[0];
|
| 273 |
+
const int sector = (i0 / 2) % sect_dims;
|
| 274 |
+
|
| 275 |
+
float theta_base = 0.0;
|
| 276 |
+
if (sector < sections.v[0]) {
|
| 277 |
+
const int p = sector;
|
| 278 |
+
theta_base = pos[i2] * dpct::pow(theta_scale, p);
|
| 279 |
+
} else if (sector >= sections.v[0] && sector < sec_w) {
|
| 280 |
+
const int p = sector - sections.v[0];
|
| 281 |
+
theta_base = pos[i2 + ne02] * dpct::pow(theta_scale, p);
|
| 282 |
+
}
|
| 283 |
+
|
| 284 |
+
const float freq_factor = has_ff ? freq_factors[i0 / 2] : 1.0f;
|
| 285 |
+
|
| 286 |
+
float cos_theta;
|
| 287 |
+
float sin_theta;
|
| 288 |
+
|
| 289 |
+
rope_yarn<forward>(theta_base / freq_factor, freq_scale, corr_dims, i0,
|
| 290 |
+
ext_factor, attn_factor, cos_theta, sin_theta);
|
| 291 |
+
|
| 292 |
+
const float x0 = x[ix + 0];
|
| 293 |
+
const float x1 = x[ix + n_dims];
|
| 294 |
+
|
| 295 |
+
dst[idst + 0] = x0 * cos_theta - x1 * sin_theta;
|
| 296 |
+
dst[idst + n_dims] = x0 * sin_theta + x1 * cos_theta;
|
| 297 |
+
}
|
| 298 |
+
|
| 299 |
+
template <bool forward, typename T, typename D>
|
| 300 |
+
static void
|
| 301 |
+
rope_norm_sycl(const T *x, D *dst, const int ne00, const int ne01,
|
| 302 |
+
const int ne02, const int s01, const int s02, const int s03,
|
| 303 |
+
const int s1, const int s2, const int s3, const int n_dims,
|
| 304 |
+
const int n_offs, const int nr, const int32_t *pos, const float freq_scale,
|
| 305 |
+
const float freq_base, const float ext_factor,
|
| 306 |
+
const float attn_factor, const rope_corr_dims corr_dims,
|
| 307 |
+
const float *freq_factors, const int64_t *row_indices,
|
| 308 |
+
const int set_rows_stride, dpct::queue_ptr stream) {
|
| 309 |
+
GGML_ASSERT(ne00 % 2 == 0);
|
| 310 |
+
const dpct::dim3 block_dims(1, SYCL_ROPE_BLOCK_SIZE, 1);
|
| 311 |
+
const int n_blocks_x =
|
| 312 |
+
(ne00 + 2 * SYCL_ROPE_BLOCK_SIZE - 1) / (2 * SYCL_ROPE_BLOCK_SIZE);
|
| 313 |
+
const dpct::dim3 block_nums(nr, n_blocks_x, 1);
|
| 314 |
+
|
| 315 |
+
const float theta_scale = powf(freq_base, -2.0f / n_dims);
|
| 316 |
+
|
| 317 |
+
if (freq_factors == nullptr) {
|
| 318 |
+
stream->parallel_for(
|
| 319 |
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
| 320 |
+
[=](sycl::nd_item<3> item_ct1) {
|
| 321 |
+
GGML_UNUSED(item_ct1);
|
| 322 |
+
rope_norm<forward, false>(
|
| 323 |
+
x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims,
|
| 324 |
+
n_offs, pos, freq_scale, ext_factor, attn_factor, corr_dims,
|
| 325 |
+
theta_scale, freq_factors, row_indices, set_rows_stride);
|
| 326 |
+
});
|
| 327 |
+
} else {
|
| 328 |
+
stream->parallel_for(
|
| 329 |
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
| 330 |
+
[=](sycl::nd_item<3> item_ct1) {
|
| 331 |
+
GGML_UNUSED(item_ct1);
|
| 332 |
+
rope_norm<forward, true>(
|
| 333 |
+
x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims,
|
| 334 |
+
n_offs, pos, freq_scale, ext_factor, attn_factor, corr_dims,
|
| 335 |
+
theta_scale, freq_factors, row_indices, set_rows_stride);
|
| 336 |
+
});
|
| 337 |
+
}
|
| 338 |
+
}
|
| 339 |
+
|
| 340 |
+
template <bool forward, typename T, typename D>
|
| 341 |
+
static void
|
| 342 |
+
rope_neox_sycl(const T *x, D *dst, const int ne00, const int ne01,
|
| 343 |
+
const int ne02, const int s01, const int s02, const int s03,
|
| 344 |
+
const int s1, const int s2, const int s3, const int n_dims,
|
| 345 |
+
const int n_offs, const int nr, const int32_t *pos, const float freq_scale,
|
| 346 |
+
const float freq_base, const float ext_factor,
|
| 347 |
+
const float attn_factor, const rope_corr_dims corr_dims,
|
| 348 |
+
const float *freq_factors, const int64_t *row_indices,
|
| 349 |
+
const int set_rows_stride, dpct::queue_ptr stream) {
|
| 350 |
+
GGML_ASSERT(ne00 % 2 == 0);
|
| 351 |
+
const dpct::dim3 block_dims(1, SYCL_ROPE_BLOCK_SIZE, 1);
|
| 352 |
+
const int n_blocks_x =
|
| 353 |
+
(ne00 + 2 * SYCL_ROPE_BLOCK_SIZE - 1) / (2 * SYCL_ROPE_BLOCK_SIZE);
|
| 354 |
+
const dpct::dim3 block_nums(nr, n_blocks_x, 1);
|
| 355 |
+
|
| 356 |
+
const float theta_scale = powf(freq_base, -2.0f / n_dims);
|
| 357 |
+
|
| 358 |
+
if (freq_factors == nullptr) {
|
| 359 |
+
stream->parallel_for(
|
| 360 |
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
| 361 |
+
[=](sycl::nd_item<3> item_ct1) {
|
| 362 |
+
GGML_UNUSED(item_ct1);
|
| 363 |
+
rope_neox<forward, false>(
|
| 364 |
+
x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims,
|
| 365 |
+
n_offs, pos, freq_scale, ext_factor, attn_factor, corr_dims,
|
| 366 |
+
theta_scale, freq_factors, row_indices, set_rows_stride);
|
| 367 |
+
});
|
| 368 |
+
} else {
|
| 369 |
+
stream->parallel_for(
|
| 370 |
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
| 371 |
+
[=](sycl::nd_item<3> item_ct1) {
|
| 372 |
+
GGML_UNUSED(item_ct1);
|
| 373 |
+
rope_neox<forward, true>(
|
| 374 |
+
x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims,
|
| 375 |
+
n_offs, pos, freq_scale, ext_factor, attn_factor, corr_dims,
|
| 376 |
+
theta_scale, freq_factors, row_indices, set_rows_stride);
|
| 377 |
+
});
|
| 378 |
+
}
|
| 379 |
+
}
|
| 380 |
+
|
| 381 |
+
template <bool forward, typename T>
|
| 382 |
+
static void
|
| 383 |
+
rope_multi_sycl(const T *x, T *dst, const int ne00, const int ne01,
|
| 384 |
+
const int ne02, const int s01, const int s02, const int s03,
|
| 385 |
+
const int s1, const int s2, const int s3, const int n_dims,
|
| 386 |
+
const int n_offs, const int nr, const int32_t *pos, const float freq_scale,
|
| 387 |
+
const float freq_base, const float ext_factor,
|
| 388 |
+
const float attn_factor, const rope_corr_dims corr_dims,
|
| 389 |
+
const float *freq_factors, const mrope_sections sections,
|
| 390 |
+
const bool is_imrope, dpct::queue_ptr stream) {
|
| 391 |
+
GGML_ASSERT(ne00 % 2 == 0);
|
| 392 |
+
const dpct::dim3 block_dims(1, SYCL_ROPE_BLOCK_SIZE, 1);
|
| 393 |
+
const int n_blocks_x =
|
| 394 |
+
(ne00 + 2 * SYCL_ROPE_BLOCK_SIZE - 1) / (2 * SYCL_ROPE_BLOCK_SIZE);
|
| 395 |
+
const dpct::dim3 block_nums(nr, n_blocks_x, 1);
|
| 396 |
+
|
| 397 |
+
const float theta_scale = powf(freq_base, -2.0f / n_dims);
|
| 398 |
+
|
| 399 |
+
if (freq_factors == nullptr) {
|
| 400 |
+
stream->parallel_for(
|
| 401 |
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
| 402 |
+
[=](sycl::nd_item<3> item_ct1) {
|
| 403 |
+
GGML_UNUSED(item_ct1);
|
| 404 |
+
rope_multi<forward, false, T>(
|
| 405 |
+
x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims,
|
| 406 |
+
n_offs, pos, freq_scale, ext_factor, attn_factor, corr_dims,
|
| 407 |
+
theta_scale, freq_factors, sections, is_imrope);
|
| 408 |
+
});
|
| 409 |
+
} else {
|
| 410 |
+
stream->parallel_for(
|
| 411 |
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
| 412 |
+
[=](sycl::nd_item<3> item_ct1) {
|
| 413 |
+
GGML_UNUSED(item_ct1);
|
| 414 |
+
rope_multi<forward, true, T>(
|
| 415 |
+
x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims,
|
| 416 |
+
n_offs, pos, freq_scale, ext_factor, attn_factor, corr_dims,
|
| 417 |
+
theta_scale, freq_factors, sections, is_imrope);
|
| 418 |
+
});
|
| 419 |
+
}
|
| 420 |
+
}
|
| 421 |
+
|
| 422 |
+
template <bool forward, typename T>
|
| 423 |
+
static void
|
| 424 |
+
rope_vision_sycl(const T *x, T *dst, const int ne00, const int ne01,
|
| 425 |
+
const int ne02, const int s01, const int s02, const int s03,
|
| 426 |
+
const int s1, const int s2, const int s3, const int n_dims,
|
| 427 |
+
const int nr, const int32_t *pos, const float freq_scale,
|
| 428 |
+
const float freq_base, const float ext_factor,
|
| 429 |
+
const float attn_factor, const rope_corr_dims corr_dims,
|
| 430 |
+
const float *freq_factors, const mrope_sections sections,
|
| 431 |
+
dpct::queue_ptr stream) {
|
| 432 |
+
GGML_ASSERT(ne00 % 2 == 0);
|
| 433 |
+
const dpct::dim3 block_dims(1, SYCL_ROPE_BLOCK_SIZE, 1);
|
| 434 |
+
const int n_blocks_x =
|
| 435 |
+
(ne00 + 2 * SYCL_ROPE_BLOCK_SIZE - 1) / (2 * SYCL_ROPE_BLOCK_SIZE);
|
| 436 |
+
const dpct::dim3 block_nums(nr, n_blocks_x, 1);
|
| 437 |
+
|
| 438 |
+
const float theta_scale = powf(freq_base, -2.0f / n_dims);
|
| 439 |
+
|
| 440 |
+
if (freq_factors == nullptr) {
|
| 441 |
+
stream->parallel_for(
|
| 442 |
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
| 443 |
+
[=](sycl::nd_item<3> item_ct1) {
|
| 444 |
+
GGML_UNUSED(item_ct1);
|
| 445 |
+
rope_vision<forward, false, T>(
|
| 446 |
+
x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims,
|
| 447 |
+
pos, freq_scale, ext_factor, attn_factor, corr_dims,
|
| 448 |
+
theta_scale, freq_factors, sections);
|
| 449 |
+
});
|
| 450 |
+
} else {
|
| 451 |
+
stream->parallel_for(
|
| 452 |
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
| 453 |
+
[=](sycl::nd_item<3> item_ct1) {
|
| 454 |
+
GGML_UNUSED(item_ct1);
|
| 455 |
+
rope_vision<forward, true, T>(
|
| 456 |
+
x, dst, ne00, ne01, ne02, s01, s02, s03, s1, s2, s3, n_dims,
|
| 457 |
+
pos, freq_scale, ext_factor, attn_factor, corr_dims,
|
| 458 |
+
theta_scale, freq_factors, sections);
|
| 459 |
+
});
|
| 460 |
+
}
|
| 461 |
+
}
|
| 462 |
+
|
| 463 |
+
template <bool forward>
|
| 464 |
+
void ggml_sycl_op_rope_impl(ggml_backend_sycl_context &ctx, ggml_tensor *dst,
|
| 465 |
+
const ggml_tensor *set_rows = nullptr) {
|
| 466 |
+
const ggml_tensor *src0 = dst->src[0];
|
| 467 |
+
const ggml_tensor *src1 = dst->src[1];
|
| 468 |
+
const ggml_tensor *src2 = dst->src[2];
|
| 469 |
+
|
| 470 |
+
const float *src0_d = (const float *)src0->data;
|
| 471 |
+
const float *src1_d = (const float *)src1->data;
|
| 472 |
+
|
| 473 |
+
void *dst_d = dst->data;
|
| 474 |
+
const int64_t *row_indices = nullptr;
|
| 475 |
+
ggml_type dst_type = dst->type;
|
| 476 |
+
int set_rows_stride = 0;
|
| 477 |
+
|
| 478 |
+
if (set_rows != nullptr) {
|
| 479 |
+
GGML_ASSERT(forward);
|
| 480 |
+
dst_d = set_rows->data;
|
| 481 |
+
row_indices = (const int64_t *)set_rows->src[1]->data;
|
| 482 |
+
dst_type = set_rows->type;
|
| 483 |
+
set_rows_stride = set_rows->nb[1] / ggml_type_size(set_rows->type);
|
| 484 |
+
}
|
| 485 |
+
dpct::queue_ptr stream = ctx.stream();
|
| 486 |
+
|
| 487 |
+
GGML_ASSERT(src0->type == GGML_TYPE_F32 || src0->type == GGML_TYPE_F16);
|
| 488 |
+
GGML_ASSERT(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16);
|
| 489 |
+
GGML_ASSERT(src0->type == dst->type ||
|
| 490 |
+
(src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16));
|
| 491 |
+
|
| 492 |
+
const int64_t ne00 = src0->ne[0]; // head dims
|
| 493 |
+
const int64_t ne01 = src0->ne[1]; // num heads
|
| 494 |
+
const int64_t ne02 = src0->ne[2]; // num heads
|
| 495 |
+
const int64_t nr = ggml_nrows(src0);
|
| 496 |
+
|
| 497 |
+
const size_t s01 = src0->nb[1] / ggml_type_size(src0->type);
|
| 498 |
+
const size_t s02 = src0->nb[2] / ggml_type_size(src0->type);
|
| 499 |
+
const size_t s03 = src0->nb[3] / ggml_type_size(src0->type);
|
| 500 |
+
|
| 501 |
+
const size_t s1 = dst->nb[1] / ggml_type_size(dst->type);
|
| 502 |
+
const size_t s2 = dst->nb[2] / ggml_type_size(dst->type);
|
| 503 |
+
const size_t s3 = dst->nb[3] / ggml_type_size(dst->type);
|
| 504 |
+
|
| 505 |
+
const int n_dims = ((int32_t *)dst->op_params)[1];
|
| 506 |
+
const int mode = ((int32_t *)dst->op_params)[2];
|
| 507 |
+
const int n_ctx_orig = ((int32_t *)dst->op_params)[4];
|
| 508 |
+
const int n_offs = ((int32_t *)dst->op_params)[15];
|
| 509 |
+
mrope_sections sections;
|
| 510 |
+
|
| 511 |
+
float freq_base;
|
| 512 |
+
float freq_scale;
|
| 513 |
+
float ext_factor;
|
| 514 |
+
float attn_factor;
|
| 515 |
+
float beta_fast;
|
| 516 |
+
float beta_slow;
|
| 517 |
+
|
| 518 |
+
memcpy(&freq_base, (int32_t *)dst->op_params + 5, sizeof(float));
|
| 519 |
+
memcpy(&freq_scale, (int32_t *)dst->op_params + 6, sizeof(float));
|
| 520 |
+
memcpy(&ext_factor, (int32_t *)dst->op_params + 7, sizeof(float));
|
| 521 |
+
memcpy(&attn_factor, (int32_t *)dst->op_params + 8, sizeof(float));
|
| 522 |
+
memcpy(&beta_fast, (int32_t *)dst->op_params + 9, sizeof(float));
|
| 523 |
+
memcpy(&beta_slow, (int32_t *)dst->op_params + 10, sizeof(float));
|
| 524 |
+
memcpy(§ions.v, (int32_t *)dst->op_params + 11, sizeof(int) * 4);
|
| 525 |
+
|
| 526 |
+
const bool is_neox = mode & GGML_ROPE_TYPE_NEOX;
|
| 527 |
+
const bool is_mrope = mode & GGML_ROPE_TYPE_MROPE;
|
| 528 |
+
const bool is_imrope = mode == GGML_ROPE_TYPE_IMROPE;
|
| 529 |
+
const bool is_vision = mode == GGML_ROPE_TYPE_VISION;
|
| 530 |
+
|
| 531 |
+
if (is_mrope) {
|
| 532 |
+
GGML_ASSERT(sections.v[0] > 0 || sections.v[1] > 0 ||
|
| 533 |
+
sections.v[2] > 0);
|
| 534 |
+
}
|
| 535 |
+
|
| 536 |
+
if (is_vision) {
|
| 537 |
+
GGML_ASSERT(n_dims == ne00 / 2);
|
| 538 |
+
GGML_ASSERT(n_offs == 0); // offset not supported for vision, as the rotated pairs span the whole row
|
| 539 |
+
}
|
| 540 |
+
|
| 541 |
+
const int32_t *pos = (const int32_t *)src1_d;
|
| 542 |
+
|
| 543 |
+
const float *freq_factors = nullptr;
|
| 544 |
+
if (src2 != nullptr) {
|
| 545 |
+
freq_factors = (const float *)src2->data;
|
| 546 |
+
}
|
| 547 |
+
|
| 548 |
+
rope_corr_dims corr_dims;
|
| 549 |
+
ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast,
|
| 550 |
+
beta_slow, corr_dims.v);
|
| 551 |
+
|
| 552 |
+
// compute
|
| 553 |
+
if (is_neox) {
|
| 554 |
+
GGML_SYCL_DEBUG("%s: neox path\n", __func__);
|
| 555 |
+
if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F32) {
|
| 556 |
+
rope_neox_sycl<forward, float, float>(
|
| 557 |
+
(const float *)src0_d, (float *)dst_d, ne00, ne01, ne02, s01,
|
| 558 |
+
s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
|
| 559 |
+
ext_factor, attn_factor, corr_dims, freq_factors, row_indices,
|
| 560 |
+
set_rows_stride, stream);
|
| 561 |
+
} else if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F16) {
|
| 562 |
+
rope_neox_sycl<forward, float, sycl::half>(
|
| 563 |
+
(const float *)src0_d, (sycl::half *)dst_d, ne00, ne01, ne02,
|
| 564 |
+
s01, s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale,
|
| 565 |
+
freq_base, ext_factor, attn_factor, corr_dims, freq_factors,
|
| 566 |
+
row_indices, set_rows_stride, stream);
|
| 567 |
+
} else if (src0->type == GGML_TYPE_F16 && dst_type == GGML_TYPE_F16) {
|
| 568 |
+
rope_neox_sycl<forward, sycl::half, sycl::half>(
|
| 569 |
+
(const sycl::half *)src0_d, (sycl::half *)dst_d, ne00, ne01,
|
| 570 |
+
ne02, s01, s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale,
|
| 571 |
+
freq_base, ext_factor, attn_factor, corr_dims, freq_factors,
|
| 572 |
+
row_indices, set_rows_stride, stream);
|
| 573 |
+
} else {
|
| 574 |
+
GGML_ABORT("Fatal error: Tensor type unsupported!");
|
| 575 |
+
}
|
| 576 |
+
} else if (is_mrope && !is_vision) {
|
| 577 |
+
GGML_SYCL_DEBUG("%s: mrope path\n", __func__);
|
| 578 |
+
if (src0->type == GGML_TYPE_F32) {
|
| 579 |
+
rope_multi_sycl<forward>((const float *)src0_d, (float *)dst_d,
|
| 580 |
+
ne00, ne01, ne02, s01, s02, s03, s1, s2,
|
| 581 |
+
s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
|
| 582 |
+
ext_factor, attn_factor, corr_dims,
|
| 583 |
+
freq_factors, sections, is_imrope, stream);
|
| 584 |
+
} else if (src0->type == GGML_TYPE_F16) {
|
| 585 |
+
rope_multi_sycl<forward>(
|
| 586 |
+
(const sycl::half *)src0_d, (sycl::half *)dst_d, ne00, ne01,
|
| 587 |
+
ne02, s01, s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale,
|
| 588 |
+
freq_base, ext_factor, attn_factor, corr_dims, freq_factors,
|
| 589 |
+
sections, is_imrope, stream);
|
| 590 |
+
} else {
|
| 591 |
+
GGML_ABORT("Fatal error: Tensor type unsupported!");
|
| 592 |
+
}
|
| 593 |
+
} else if (is_vision) {
|
| 594 |
+
GGML_SYCL_DEBUG("%s: vision path\n", __func__);
|
| 595 |
+
if (src0->type == GGML_TYPE_F32) {
|
| 596 |
+
rope_vision_sycl<forward>(
|
| 597 |
+
(const float *)src0_d, (float *)dst_d, ne00, ne01, ne02, s01,
|
| 598 |
+
s02, s03, s1, s2, s3, n_dims, nr, pos, freq_scale, freq_base,
|
| 599 |
+
ext_factor, attn_factor, corr_dims, freq_factors, sections,
|
| 600 |
+
stream);
|
| 601 |
+
} else if (src0->type == GGML_TYPE_F16) {
|
| 602 |
+
rope_vision_sycl<forward>(
|
| 603 |
+
(const sycl::half *)src0_d, (sycl::half *)dst_d, ne00, ne01,
|
| 604 |
+
ne02, s01, s02, s03, s1, s2, s3, n_dims, nr, pos, freq_scale,
|
| 605 |
+
freq_base, ext_factor, attn_factor, corr_dims, freq_factors,
|
| 606 |
+
sections, stream);
|
| 607 |
+
} else {
|
| 608 |
+
GGML_ABORT("Fatal error: Tensor type unsupported!");
|
| 609 |
+
}
|
| 610 |
+
} else {
|
| 611 |
+
GGML_SYCL_DEBUG("%s: norm path\n", __func__);
|
| 612 |
+
if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F32) {
|
| 613 |
+
rope_norm_sycl<forward, float, float>(
|
| 614 |
+
(const float *)src0_d, (float *)dst_d, ne00, ne01, ne02, s01,
|
| 615 |
+
s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale, freq_base,
|
| 616 |
+
ext_factor, attn_factor, corr_dims, freq_factors, row_indices,
|
| 617 |
+
set_rows_stride, stream);
|
| 618 |
+
} else if (src0->type == GGML_TYPE_F32 && dst_type == GGML_TYPE_F16) {
|
| 619 |
+
rope_norm_sycl<forward, float, sycl::half>(
|
| 620 |
+
(const float *)src0_d, (sycl::half *)dst_d, ne00, ne01, ne02,
|
| 621 |
+
s01, s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale,
|
| 622 |
+
freq_base, ext_factor, attn_factor, corr_dims, freq_factors,
|
| 623 |
+
row_indices, set_rows_stride, stream);
|
| 624 |
+
} else if (src0->type == GGML_TYPE_F16 && dst_type == GGML_TYPE_F16) {
|
| 625 |
+
rope_norm_sycl<forward, sycl::half, sycl::half>(
|
| 626 |
+
(const sycl::half *)src0_d, (sycl::half *)dst_d, ne00, ne01,
|
| 627 |
+
ne02, s01, s02, s03, s1, s2, s3, n_dims, n_offs, nr, pos, freq_scale,
|
| 628 |
+
freq_base, ext_factor, attn_factor, corr_dims, freq_factors,
|
| 629 |
+
row_indices, set_rows_stride, stream);
|
| 630 |
+
} else {
|
| 631 |
+
GGML_ABORT("Fatal error: Tensor type unsupported!");
|
| 632 |
+
}
|
| 633 |
+
}
|
| 634 |
+
}
|
| 635 |
+
|
| 636 |
+
void ggml_sycl_rope(ggml_backend_sycl_context &ctx, ggml_tensor *dst) {
|
| 637 |
+
scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/3);
|
| 638 |
+
|
| 639 |
+
ggml_sycl_op_rope_impl<true>(ctx, dst);
|
| 640 |
+
}
|
| 641 |
+
|
| 642 |
+
void ggml_sycl_rope_back(ggml_backend_sycl_context &ctx, ggml_tensor *dst) {
|
| 643 |
+
scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/3);
|
| 644 |
+
ggml_sycl_op_rope_impl<false>(ctx, dst);
|
| 645 |
+
}
|
| 646 |
+
|
| 647 |
+
void ggml_sycl_rope_fused(ggml_backend_sycl_context &ctx, ggml_tensor *rope,
|
| 648 |
+
ggml_tensor *set_rows) {
|
| 649 |
+
scope_op_debug_print scope_dbg_print(__func__, rope, /*num_src=*/3);
|
| 650 |
+
ggml_sycl_op_rope_impl<true>(ctx, rope, set_rows);
|
| 651 |
+
}
|
ggml/src/ggml-sycl/rope.hpp
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//
|
| 2 |
+
// MIT license
|
| 3 |
+
// Copyright (C) 2024 Intel Corporation
|
| 4 |
+
// SPDX-License-Identifier: MIT
|
| 5 |
+
//
|
| 6 |
+
|
| 7 |
+
//
|
| 8 |
+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
| 9 |
+
// See https://llvm.org/LICENSE.txt for license information.
|
| 10 |
+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
| 11 |
+
//
|
| 12 |
+
|
| 13 |
+
#ifndef GGML_SYCL_ROPE_HPP
|
| 14 |
+
#define GGML_SYCL_ROPE_HPP
|
| 15 |
+
|
| 16 |
+
#include "common.hpp"
|
| 17 |
+
|
| 18 |
+
#define SYCL_ROPE_BLOCK_SIZE 256
|
| 19 |
+
|
| 20 |
+
void ggml_sycl_rope(ggml_backend_sycl_context & ctx, ggml_tensor *dst);
|
| 21 |
+
|
| 22 |
+
void ggml_sycl_rope_back(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
|
| 23 |
+
|
| 24 |
+
void ggml_sycl_rope_fused(ggml_backend_sycl_context & ctx, ggml_tensor * dst, ggml_tensor * set_rows);
|
| 25 |
+
|
| 26 |
+
#endif // GGML_SYCL_ROPE_HPP
|
ggml/src/ggml-sycl/set.cpp
ADDED
|
@@ -0,0 +1,73 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#include "presets.hpp"
|
| 2 |
+
#include "common.hpp"
|
| 3 |
+
#include "ggml.h"
|
| 4 |
+
#include "set.hpp"
|
| 5 |
+
#include <cstdint>
|
| 6 |
+
#include <sycl/sycl.hpp>
|
| 7 |
+
using namespace sycl;
|
| 8 |
+
|
| 9 |
+
// Internal function: perform element-wise set operation for each thread
|
| 10 |
+
inline void set_f32(const float* src, float* dst,
|
| 11 |
+
const int64_t ne0, const int64_t ne1,
|
| 12 |
+
const int64_t ne2, const int64_t ne3,
|
| 13 |
+
const int64_t nb[3], const int64_t src_nb[3],
|
| 14 |
+
const int64_t offset_elem,
|
| 15 |
+
const nd_item<1>& item)
|
| 16 |
+
{
|
| 17 |
+
const size_t idx = item.get_global_id(0);
|
| 18 |
+
const size_t total = ne0 * ne1 * ne2 * ne3;
|
| 19 |
+
if (idx >= total) return;
|
| 20 |
+
|
| 21 |
+
// Convert linear index to 4D indices
|
| 22 |
+
const size_t i3 = idx / (ne2 * ne1 * ne0);
|
| 23 |
+
const size_t rem = idx % (ne2 * ne1 * ne0);
|
| 24 |
+
const size_t i2 = rem / (ne1 * ne0);
|
| 25 |
+
const size_t rem2 = rem % (ne1 * ne0);
|
| 26 |
+
const size_t i1 = rem2 / ne0;
|
| 27 |
+
const size_t i0 = rem2 % ne0;
|
| 28 |
+
|
| 29 |
+
// Compute source and destination indices and copy
|
| 30 |
+
dst[i0 + i1*nb[0] + i2*nb[1] + i3*nb[2] + offset_elem] =
|
| 31 |
+
src[i0 + i1*src_nb[0] + i2*src_nb[1] + i3*src_nb[2]];
|
| 32 |
+
}
|
| 33 |
+
|
| 34 |
+
// Main function: prepare GPU queue and launch parallel_for
|
| 35 |
+
void ggml_sycl_op_set(ggml_backend_sycl_context& ctx, ggml_tensor* dst) {
|
| 36 |
+
const ggml_tensor* src0 = dst->src[0];
|
| 37 |
+
const ggml_tensor* src1 = dst->src[1];
|
| 38 |
+
|
| 39 |
+
// Ensure shapes and types are compatible
|
| 40 |
+
GGML_ASSERT(ggml_are_same_shape(src0, dst));
|
| 41 |
+
GGML_ASSERT(ggml_is_contiguous(dst) && ggml_is_contiguous(src0));
|
| 42 |
+
GGML_ASSERT(dst->type == src0->type && src0->type == src1->type && dst->type == GGML_TYPE_F32);
|
| 43 |
+
|
| 44 |
+
const int32_t* opts = (const int32_t*) dst->op_params;
|
| 45 |
+
const int64_t nb[3] = {opts[0]/sizeof(float), opts[1]/sizeof(float), opts[2]/sizeof(float)};
|
| 46 |
+
const int64_t offset_elem = opts[3] / sizeof(float);
|
| 47 |
+
const bool inplace = opts[4];
|
| 48 |
+
|
| 49 |
+
float* dst_ptr = (float*) dst->data;
|
| 50 |
+
const float* src0_ptr = (const float*) src0->data;
|
| 51 |
+
const float* src1_ptr = (const float*) src1->data;
|
| 52 |
+
|
| 53 |
+
queue_ptr stream = ctx.stream();
|
| 54 |
+
|
| 55 |
+
// Copy src0 to dst if not inplace
|
| 56 |
+
if (!inplace)
|
| 57 |
+
stream->memcpy(dst_ptr, src0_ptr, ggml_nbytes(dst));
|
| 58 |
+
|
| 59 |
+
const int64_t ne[4] = {src1->ne[0], src1->ne[1], src1->ne[2], src1->ne[3]};
|
| 60 |
+
const int64_t src_nb[3] = {src1->nb[1]/sizeof(float), src1->nb[2]/sizeof(float), src1->nb[3]/sizeof(float)};
|
| 61 |
+
|
| 62 |
+
const size_t total_threads = ne[0]*ne[1]*ne[2]*ne[3];
|
| 63 |
+
const size_t grid_size = ((total_threads + SYCL_SET_BLOCK_SIZE - 1) / SYCL_SET_BLOCK_SIZE) * SYCL_SET_BLOCK_SIZE;
|
| 64 |
+
|
| 65 |
+
// Copy src0 to dst if not inplace
|
| 66 |
+
stream->parallel_for(
|
| 67 |
+
nd_range<1>(range<1>(grid_size), range<1>(SYCL_SET_BLOCK_SIZE)),
|
| 68 |
+
[=](nd_item<1> item) {
|
| 69 |
+
set_f32(src1_ptr, dst_ptr,
|
| 70 |
+
ne[0], ne[1], ne[2], ne[3],
|
| 71 |
+
nb, src_nb, offset_elem, item); }
|
| 72 |
+
);
|
| 73 |
+
}
|
ggml/src/ggml-sycl/set.hpp
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#pragma once
|
| 2 |
+
#include "backend.hpp"
|
| 3 |
+
#include "ggml.h"
|
| 4 |
+
|
| 5 |
+
void ggml_sycl_op_set(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
|
ggml/src/ggml-sycl/set_rows.cpp
ADDED
|
@@ -0,0 +1,577 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#include "set_rows.hpp"
|
| 2 |
+
#include "cpy.hpp"
|
| 3 |
+
|
| 4 |
+
#include "ggml-quants.h"
|
| 5 |
+
|
| 6 |
+
#include <vector>
|
| 7 |
+
|
| 8 |
+
namespace utils {
|
| 9 |
+
template<typename T>
|
| 10 |
+
static constexpr bool is_arithmetic_v() {
|
| 11 |
+
return std::is_arithmetic_v<T> || std::is_same_v<T, sycl::half>
|
| 12 |
+
#ifdef GGML_SYCL_HAS_BF16
|
| 13 |
+
|| std::is_same_v<T, sycl::ext::oneapi::bfloat16>
|
| 14 |
+
#endif
|
| 15 |
+
;
|
| 16 |
+
}
|
| 17 |
+
}
|
| 18 |
+
|
| 19 |
+
template<typename TIn, typename TOut>
|
| 20 |
+
static inline std::enable_if_t<utils::is_arithmetic_v<TIn>() && utils::is_arithmetic_v<TOut>(), void>
|
| 21 |
+
convert (const char* src, char* dst) {
|
| 22 |
+
auto src_val = *reinterpret_cast<const TIn*>(src);
|
| 23 |
+
auto dst_val = sycl::vec<TIn, 1>(src_val).template convert<TOut, sycl::rounding_mode::automatic>()[0];
|
| 24 |
+
*reinterpret_cast<TOut*>(dst) = dst_val;
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
#ifdef GGML_SYCL_HAS_BF16
|
| 28 |
+
// sycl::vec::convert does not provide a half -> bfloat16 path, so route through float.
|
| 29 |
+
template<>
|
| 30 |
+
inline void convert<sycl::half, sycl::ext::oneapi::bfloat16>(const char* src, char* dst) {
|
| 31 |
+
const float tmp = sycl::vec<sycl::half, 1>(*reinterpret_cast<const sycl::half*>(src))
|
| 32 |
+
.template convert<float, sycl::rounding_mode::automatic>()[0];
|
| 33 |
+
*reinterpret_cast<sycl::ext::oneapi::bfloat16*>(dst) = sycl::ext::oneapi::bfloat16(tmp);
|
| 34 |
+
}
|
| 35 |
+
#endif
|
| 36 |
+
|
| 37 |
+
template <typename TIn, typename TIdx, typename blockType, int qk, cpy_kernel_t cpyblck>
|
| 38 |
+
static void set_rows_sycl_q(const char * __restrict__ src0_d,
|
| 39 |
+
const TIdx * __restrict__ src1_d,
|
| 40 |
+
blockType * __restrict__ dst_d,
|
| 41 |
+
// tensor dimensions src0 and src1
|
| 42 |
+
const int64_t ne00,
|
| 43 |
+
const int64_t ne01,
|
| 44 |
+
const int64_t ne02,
|
| 45 |
+
const int64_t ne03,
|
| 46 |
+
const int64_t ne10,
|
| 47 |
+
const int64_t ne11,
|
| 48 |
+
const int64_t ne12,
|
| 49 |
+
const int64_t ne13,
|
| 50 |
+
// strides for src0
|
| 51 |
+
const size_t nb00,
|
| 52 |
+
const size_t nb01,
|
| 53 |
+
const size_t nb02,
|
| 54 |
+
const size_t nb03,
|
| 55 |
+
// strides for src1
|
| 56 |
+
const size_t nb10,
|
| 57 |
+
const size_t nb11,
|
| 58 |
+
const size_t nb12,
|
| 59 |
+
const size_t nb13,
|
| 60 |
+
// strides for dst
|
| 61 |
+
const size_t nb1,
|
| 62 |
+
const size_t nb2,
|
| 63 |
+
const size_t nb3,
|
| 64 |
+
queue_ptr stream) {
|
| 65 |
+
const int64_t total_blocks = (ne00 * ne01 * ne02 * ne03) / qk;
|
| 66 |
+
constexpr int block_size = 256;
|
| 67 |
+
const int64_t grid_size = ceil_div(total_blocks, block_size);
|
| 68 |
+
|
| 69 |
+
stream->parallel_for(sycl::nd_range<1>(grid_size * block_size, block_size), [=](sycl::nd_item<1> item_ct1) {
|
| 70 |
+
const int64_t i = item_ct1.get_global_linear_id();
|
| 71 |
+
if (i >= total_blocks) {
|
| 72 |
+
return;
|
| 73 |
+
}
|
| 74 |
+
const int64_t i_base = i * qk;
|
| 75 |
+
const int64_t i03 = i_base / (ne00 * ne01 * ne02);
|
| 76 |
+
const int64_t rem1 = i_base - i03 * (ne00 * ne01 * ne02);
|
| 77 |
+
const int64_t i02 = rem1 / (ne00 * ne01);
|
| 78 |
+
const int64_t rem2 = rem1 - i02 * ne00 * ne01;
|
| 79 |
+
const int64_t i01 = rem2 / ne00;
|
| 80 |
+
const int64_t i00 = rem2 - i01 * ne00;
|
| 81 |
+
const int64_t i12 = i03 % ne12;
|
| 82 |
+
const int64_t i11 = i02 % ne11;
|
| 83 |
+
const int64_t i10 = i01;
|
| 84 |
+
const size_t src_offset = calculate_offset<3>({ nb01, nb02, nb03 }, { i01, i02, i03 });
|
| 85 |
+
const char * src_block = src0_d + src_offset + i00 * sizeof(TIn);
|
| 86 |
+
const size_t src1_offset = calculate_offset<3>({ nb10, nb11, nb12 }, { i10, i11, i12 });
|
| 87 |
+
const int64_t dst_row = src1_d[src1_offset / sizeof(TIdx)];
|
| 88 |
+
const size_t dst_offset =
|
| 89 |
+
calculate_offset<3>({ nb1, nb2, nb3 }, { dst_row, i02, i03 }) + (i00 / qk) * sizeof(blockType);
|
| 90 |
+
char * dst_block = reinterpret_cast<char *>(reinterpret_cast<char *>(dst_d) + dst_offset);
|
| 91 |
+
if constexpr (std::is_same_v<TIn, float>) {
|
| 92 |
+
cpyblck(src_block, dst_block);
|
| 93 |
+
} else {
|
| 94 |
+
float src_block_f32[qk];
|
| 95 |
+
const TIn * src_block_t = reinterpret_cast<const TIn *>(src_block);
|
| 96 |
+
for (int j = 0; j < qk; ++j) {
|
| 97 |
+
src_block_f32[j] = (float) src_block_t[j];
|
| 98 |
+
}
|
| 99 |
+
cpyblck(reinterpret_cast<const char *>(src_block_f32), dst_block);
|
| 100 |
+
}
|
| 101 |
+
});
|
| 102 |
+
GGML_UNUSED(ne10);
|
| 103 |
+
GGML_UNUSED(ne13);
|
| 104 |
+
GGML_UNUSED(nb00);
|
| 105 |
+
GGML_UNUSED(nb13);
|
| 106 |
+
}
|
| 107 |
+
|
| 108 |
+
template<typename blockType>
|
| 109 |
+
using quantize_row_qk_t = void (*)(const float *, blockType *, int64_t);
|
| 110 |
+
|
| 111 |
+
using quantize_rows_f_t = size_t (*)(const float *, void *, int64_t, int64_t, const float *);
|
| 112 |
+
|
| 113 |
+
template <typename TIn, typename TIdx, typename blockType, int qk, quantize_row_qk_t<blockType> quantize_row>
|
| 114 |
+
static void set_rows_sycl_qk_host(
|
| 115 |
+
const ggml_tensor * src0,
|
| 116 |
+
const ggml_tensor * src1,
|
| 117 |
+
ggml_tensor * dst,
|
| 118 |
+
const int64_t ne00,
|
| 119 |
+
const int64_t ne01,
|
| 120 |
+
const int64_t ne02,
|
| 121 |
+
const int64_t ne03,
|
| 122 |
+
const int64_t ne11,
|
| 123 |
+
const int64_t ne12,
|
| 124 |
+
const size_t nb01,
|
| 125 |
+
const size_t nb02,
|
| 126 |
+
const size_t nb03,
|
| 127 |
+
const size_t nb10,
|
| 128 |
+
const size_t nb11,
|
| 129 |
+
const size_t nb12,
|
| 130 |
+
const size_t nb1,
|
| 131 |
+
const size_t nb2,
|
| 132 |
+
const size_t nb3,
|
| 133 |
+
queue_ptr stream) {
|
| 134 |
+
GGML_ASSERT(ne00 % qk == 0);
|
| 135 |
+
|
| 136 |
+
const size_t src0_bytes = ggml_nbytes(src0);
|
| 137 |
+
const size_t src1_bytes = ggml_nbytes(src1);
|
| 138 |
+
|
| 139 |
+
std::vector<char> src0_host(src0_bytes);
|
| 140 |
+
std::vector<char> src1_host(src1_bytes);
|
| 141 |
+
|
| 142 |
+
stream->memcpy(src0_host.data(), src0->data, src0_bytes);
|
| 143 |
+
stream->memcpy(src1_host.data(), src1->data, src1_bytes);
|
| 144 |
+
stream->wait();
|
| 145 |
+
|
| 146 |
+
std::vector<float> src_row_f32(ne00);
|
| 147 |
+
const int64_t nblocks = ne00 / qk;
|
| 148 |
+
std::vector<blockType> dst_row_q(nblocks);
|
| 149 |
+
|
| 150 |
+
for (int64_t i03 = 0; i03 < ne03; ++i03) {
|
| 151 |
+
for (int64_t i02 = 0; i02 < ne02; ++i02) {
|
| 152 |
+
for (int64_t i01 = 0; i01 < ne01; ++i01) {
|
| 153 |
+
const int64_t i12 = i03 % ne12;
|
| 154 |
+
const int64_t i11 = i02 % ne11;
|
| 155 |
+
const int64_t i10 = i01;
|
| 156 |
+
|
| 157 |
+
const size_t src1_offset = calculate_offset<3>({ nb10, nb11, nb12 }, { i10, i11, i12 });
|
| 158 |
+
const int64_t dst_row = *(const TIdx *) (src1_host.data() + src1_offset);
|
| 159 |
+
|
| 160 |
+
const size_t src0_row_offset = calculate_offset<3>({ nb01, nb02, nb03 }, { i01, i02, i03 });
|
| 161 |
+
const TIn * src_row = reinterpret_cast<const TIn *>(src0_host.data() + src0_row_offset);
|
| 162 |
+
|
| 163 |
+
for (int64_t i00 = 0; i00 < ne00; ++i00) {
|
| 164 |
+
src_row_f32[i00] = (float) src_row[i00];
|
| 165 |
+
}
|
| 166 |
+
|
| 167 |
+
quantize_row(src_row_f32.data(), dst_row_q.data(), ne00);
|
| 168 |
+
|
| 169 |
+
const size_t dst_offset = calculate_offset<3>({ nb1, nb2, nb3 }, { dst_row, i02, i03 });
|
| 170 |
+
stream->memcpy((char *) dst->data + dst_offset, dst_row_q.data(), nblocks * sizeof(blockType));
|
| 171 |
+
stream->wait();
|
| 172 |
+
}
|
| 173 |
+
}
|
| 174 |
+
}
|
| 175 |
+
}
|
| 176 |
+
|
| 177 |
+
template <typename TIn, typename TIdx, typename blockType, int qk, quantize_rows_f_t quantize_rows>
|
| 178 |
+
static void set_rows_sycl_iq_host(
|
| 179 |
+
const ggml_tensor * src0,
|
| 180 |
+
const ggml_tensor * src1,
|
| 181 |
+
ggml_tensor * dst,
|
| 182 |
+
const int64_t ne00,
|
| 183 |
+
const int64_t ne01,
|
| 184 |
+
const int64_t ne02,
|
| 185 |
+
const int64_t ne03,
|
| 186 |
+
const int64_t ne11,
|
| 187 |
+
const int64_t ne12,
|
| 188 |
+
const size_t nb01,
|
| 189 |
+
const size_t nb02,
|
| 190 |
+
const size_t nb03,
|
| 191 |
+
const size_t nb10,
|
| 192 |
+
const size_t nb11,
|
| 193 |
+
const size_t nb12,
|
| 194 |
+
const size_t nb1,
|
| 195 |
+
const size_t nb2,
|
| 196 |
+
const size_t nb3,
|
| 197 |
+
queue_ptr stream) {
|
| 198 |
+
GGML_ASSERT(ne00 % qk == 0);
|
| 199 |
+
|
| 200 |
+
const size_t src0_bytes = ggml_nbytes(src0);
|
| 201 |
+
const size_t src1_bytes = ggml_nbytes(src1);
|
| 202 |
+
|
| 203 |
+
std::vector<char> src0_host(src0_bytes);
|
| 204 |
+
std::vector<char> src1_host(src1_bytes);
|
| 205 |
+
|
| 206 |
+
stream->memcpy(src0_host.data(), src0->data, src0_bytes);
|
| 207 |
+
stream->memcpy(src1_host.data(), src1->data, src1_bytes);
|
| 208 |
+
stream->wait();
|
| 209 |
+
|
| 210 |
+
std::vector<float> src_row_f32(ne00);
|
| 211 |
+
const int64_t nblocks = ne00 / qk;
|
| 212 |
+
std::vector<blockType> dst_row_q(nblocks);
|
| 213 |
+
|
| 214 |
+
for (int64_t i03 = 0; i03 < ne03; ++i03) {
|
| 215 |
+
for (int64_t i02 = 0; i02 < ne02; ++i02) {
|
| 216 |
+
for (int64_t i01 = 0; i01 < ne01; ++i01) {
|
| 217 |
+
const int64_t i12 = i03 % ne12;
|
| 218 |
+
const int64_t i11 = i02 % ne11;
|
| 219 |
+
const int64_t i10 = i01;
|
| 220 |
+
|
| 221 |
+
const size_t src1_offset = calculate_offset<3>({ nb10, nb11, nb12 }, { i10, i11, i12 });
|
| 222 |
+
const int64_t dst_row = *(const TIdx *) (src1_host.data() + src1_offset);
|
| 223 |
+
|
| 224 |
+
const size_t src0_row_offset = calculate_offset<3>({ nb01, nb02, nb03 }, { i01, i02, i03 });
|
| 225 |
+
const TIn * src_row = reinterpret_cast<const TIn *>(src0_host.data() + src0_row_offset);
|
| 226 |
+
|
| 227 |
+
for (int64_t i00 = 0; i00 < ne00; ++i00) {
|
| 228 |
+
src_row_f32[i00] = (float) src_row[i00];
|
| 229 |
+
}
|
| 230 |
+
|
| 231 |
+
quantize_rows(src_row_f32.data(), dst_row_q.data(), 1, ne00, nullptr);
|
| 232 |
+
|
| 233 |
+
const size_t dst_offset = calculate_offset<3>({ nb1, nb2, nb3 }, { dst_row, i02, i03 });
|
| 234 |
+
stream->memcpy((char *) dst->data + dst_offset, dst_row_q.data(), nblocks * sizeof(blockType));
|
| 235 |
+
stream->wait();
|
| 236 |
+
}
|
| 237 |
+
}
|
| 238 |
+
}
|
| 239 |
+
}
|
| 240 |
+
|
| 241 |
+
template<typename TIn, typename TIdx, typename TOut>
|
| 242 |
+
static void k_set_rows(
|
| 243 |
+
const char * __restrict__ src0, const TIdx * __restrict__ src1, char * __restrict__ dst,
|
| 244 |
+
const int64_t ne00, const int64_t ne01, const int64_t ne02,
|
| 245 |
+
const int64_t ne11, const int64_t ne12,
|
| 246 |
+
const size_t nb01, const size_t nb02, const size_t nb03,
|
| 247 |
+
const size_t nb10, const size_t nb11, const size_t nb12,
|
| 248 |
+
const size_t nb1, const size_t nb2, const size_t nb3,
|
| 249 |
+
const size_t src_type_size, const size_t dst_type_size,
|
| 250 |
+
const int64_t total_elements,
|
| 251 |
+
const sycl::nd_item<1> & item_ct1) {
|
| 252 |
+
|
| 253 |
+
const int64_t i = item_ct1.get_global_linear_id();
|
| 254 |
+
if (i >= total_elements) {
|
| 255 |
+
return;
|
| 256 |
+
}
|
| 257 |
+
|
| 258 |
+
const int64_t i03 = i / (ne00 * ne01 * ne02);
|
| 259 |
+
const int64_t i02 = (i - i03 * ne00 * ne01 * ne02) / (ne00 * ne01);
|
| 260 |
+
const int64_t i01 = (i - i03 * ne00 * ne01 * ne02 - i02 * ne00 * ne01) / ne00;
|
| 261 |
+
const int64_t i00 = i - i03 * ne00 * ne01 * ne02 - i02 * ne00 * ne01 - i01 * ne00;
|
| 262 |
+
|
| 263 |
+
const int64_t i12 = i03 % ne12;
|
| 264 |
+
const int64_t i11 = i02 % ne11;
|
| 265 |
+
const int64_t i10 = i01;
|
| 266 |
+
|
| 267 |
+
const int64_t dst_row = *(const TIdx *)((const char *)src1 + calculate_offset<3>({nb10, nb11, nb12}, {i10, i11, i12}));
|
| 268 |
+
|
| 269 |
+
const char * src0_row = src0 + calculate_offset<3>({nb01, nb02, nb03}, {i01, i02, i03});
|
| 270 |
+
const char * src_elem = src0_row + i00 * src_type_size;
|
| 271 |
+
char * dst_row_ptr = dst + dst_row*nb1 + i02*nb2 + i03*nb3;
|
| 272 |
+
char * dst_elem = dst_row_ptr + i00 * dst_type_size;
|
| 273 |
+
|
| 274 |
+
convert<TIn, TOut>(src_elem, dst_elem);
|
| 275 |
+
}
|
| 276 |
+
|
| 277 |
+
template<typename TIn, typename TIdx, typename TOut>
|
| 278 |
+
static void set_rows_sycl(
|
| 279 |
+
const char * src0_d, const TIdx * src1_d, char * dst_d,
|
| 280 |
+
const int64_t ne00, const int64_t ne01, const int64_t ne02, const int64_t ne03,
|
| 281 |
+
const int64_t ne11, const int64_t ne12, const size_t nb01, const size_t nb02, const size_t nb03,
|
| 282 |
+
const size_t nb10, const size_t nb11, const size_t nb12,
|
| 283 |
+
const size_t nb1, const size_t nb2, const size_t nb3,
|
| 284 |
+
const size_t src_type_size, const size_t dst_type_size,
|
| 285 |
+
queue_ptr stream) {
|
| 286 |
+
|
| 287 |
+
const int64_t total_elements = ne00 * ne01 * ne02 * ne03;
|
| 288 |
+
|
| 289 |
+
constexpr int block_size = 64;
|
| 290 |
+
const int64_t grid_size = ceil_div(total_elements, block_size);
|
| 291 |
+
|
| 292 |
+
stream->parallel_for(
|
| 293 |
+
sycl::nd_range<1>(grid_size * block_size, block_size),
|
| 294 |
+
[=](sycl::nd_item<1> item_ct1) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
| 295 |
+
k_set_rows<TIn, TIdx, TOut>(
|
| 296 |
+
src0_d, src1_d, dst_d,
|
| 297 |
+
ne00, ne01, ne02,
|
| 298 |
+
ne11, ne12,
|
| 299 |
+
nb01, nb02, nb03,
|
| 300 |
+
nb10, nb11, nb12,
|
| 301 |
+
nb1, nb2, nb3,
|
| 302 |
+
src_type_size, dst_type_size,
|
| 303 |
+
total_elements,
|
| 304 |
+
item_ct1
|
| 305 |
+
);
|
| 306 |
+
}
|
| 307 |
+
);
|
| 308 |
+
}
|
| 309 |
+
|
| 310 |
+
template<typename TIn, typename TIdx>
|
| 311 |
+
static void set_rows_sycl(ggml_backend_sycl_context & ctx, const ggml_tensor * src0, const ggml_tensor * src1, ggml_tensor * dst) {
|
| 312 |
+
const char * src0_d = (const char *)src0->data;
|
| 313 |
+
const TIdx * src1_d = (const TIdx *)src1->data;
|
| 314 |
+
|
| 315 |
+
GGML_TENSOR_BINARY_OP_LOCALS
|
| 316 |
+
|
| 317 |
+
dpct::queue_ptr stream = ctx.stream();
|
| 318 |
+
switch (dst->type) {
|
| 319 |
+
case GGML_TYPE_F32:
|
| 320 |
+
set_rows_sycl<TIn, TIdx, float>(
|
| 321 |
+
src0_d, src1_d, (char *)dst->data,
|
| 322 |
+
ne00, ne01, ne02, ne03,
|
| 323 |
+
ne11, ne12,
|
| 324 |
+
nb01, nb02, nb03,
|
| 325 |
+
nb10, nb11, nb12,
|
| 326 |
+
nb1, nb2, nb3,
|
| 327 |
+
sizeof(TIn), sizeof(float),
|
| 328 |
+
stream
|
| 329 |
+
);
|
| 330 |
+
break;
|
| 331 |
+
case GGML_TYPE_F16:
|
| 332 |
+
dpct::has_capability_or_fail(stream->get_device(), { sycl::aspect::fp16 });
|
| 333 |
+
set_rows_sycl<TIn, TIdx, sycl::half>(
|
| 334 |
+
src0_d, src1_d, (char *)dst->data,
|
| 335 |
+
ne00, ne01, ne02, ne03,
|
| 336 |
+
ne11, ne12,
|
| 337 |
+
nb01, nb02, nb03,
|
| 338 |
+
nb10, nb11, nb12,
|
| 339 |
+
nb1, nb2, nb3,
|
| 340 |
+
sizeof(TIn), sizeof(sycl::half),
|
| 341 |
+
stream
|
| 342 |
+
);
|
| 343 |
+
break;
|
| 344 |
+
#ifdef GGML_SYCL_HAS_BF16
|
| 345 |
+
case GGML_TYPE_BF16:
|
| 346 |
+
set_rows_sycl<TIn, TIdx, sycl::ext::oneapi::bfloat16>(
|
| 347 |
+
src0_d, src1_d, (char *)dst->data,
|
| 348 |
+
ne00, ne01, ne02, ne03,
|
| 349 |
+
ne11, ne12,
|
| 350 |
+
nb01, nb02, nb03,
|
| 351 |
+
nb10, nb11, nb12,
|
| 352 |
+
nb1, nb2, nb3,
|
| 353 |
+
sizeof(TIn), sizeof(sycl::ext::oneapi::bfloat16),
|
| 354 |
+
stream
|
| 355 |
+
);
|
| 356 |
+
break;
|
| 357 |
+
#endif
|
| 358 |
+
case GGML_TYPE_Q8_0:
|
| 359 |
+
set_rows_sycl_q<TIn, TIdx, block_q8_0, QK8_0, cpy_blck_f32_q8_0>(
|
| 360 |
+
src0_d, src1_d, (block_q8_0 *) dst->data, ne00, ne01, ne02, ne03,
|
| 361 |
+
ne10, ne11, ne12, ne13, nb00, nb01,
|
| 362 |
+
nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
|
| 363 |
+
break;
|
| 364 |
+
case GGML_TYPE_Q1_0:
|
| 365 |
+
set_rows_sycl_q<TIn, TIdx, block_q1_0, QK1_0, cpy_blck_f32_q1_0>(
|
| 366 |
+
src0_d, src1_d, (block_q1_0 *) dst->data, ne00, ne01, ne02, ne03,
|
| 367 |
+
ne10, ne11, ne12, ne13, nb00, nb01,
|
| 368 |
+
nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
|
| 369 |
+
break;
|
| 370 |
+
case GGML_TYPE_Q2_0:
|
| 371 |
+
set_rows_sycl_q<TIn, TIdx, block_q2_0, QK2_0, cpy_blck_f32_q2_0>(
|
| 372 |
+
src0_d, src1_d, (block_q2_0 *) dst->data, ne00, ne01, ne02, ne03,
|
| 373 |
+
ne10, ne11, ne12, ne13, nb00, nb01,
|
| 374 |
+
nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
|
| 375 |
+
break;
|
| 376 |
+
case GGML_TYPE_Q5_1:
|
| 377 |
+
set_rows_sycl_q<TIn, TIdx, block_q5_1, QK5_1, cpy_blck_f32_q5_1>(
|
| 378 |
+
src0_d, src1_d, (block_q5_1 *) dst->data, ne00, ne01, ne02, ne03,
|
| 379 |
+
ne10, ne11, ne12, ne13, nb00, nb01,
|
| 380 |
+
nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
|
| 381 |
+
break;
|
| 382 |
+
case GGML_TYPE_Q5_0:
|
| 383 |
+
set_rows_sycl_q<TIn, TIdx, block_q5_0, QK5_0, cpy_blck_f32_q5_0>(
|
| 384 |
+
src0_d, src1_d, (block_q5_0 *) dst->data, ne00, ne01, ne02, ne03,
|
| 385 |
+
ne10, ne11, ne12, ne13, nb00, nb01,
|
| 386 |
+
nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
|
| 387 |
+
break;
|
| 388 |
+
case GGML_TYPE_Q4_1:
|
| 389 |
+
set_rows_sycl_q<TIn, TIdx, block_q4_1, QK4_1, cpy_blck_f32_q4_1>(
|
| 390 |
+
src0_d, src1_d, (block_q4_1 *) dst->data, ne00, ne01, ne02, ne03,
|
| 391 |
+
ne10, ne11, ne12, ne13, nb00, nb01,
|
| 392 |
+
nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
|
| 393 |
+
break;
|
| 394 |
+
case GGML_TYPE_Q4_0:
|
| 395 |
+
set_rows_sycl_q<TIn, TIdx, block_q4_0, QK4_0, cpy_blck_f32_q4_0>(
|
| 396 |
+
src0_d, src1_d, (block_q4_0 *) dst->data, ne00, ne01, ne02, ne03,
|
| 397 |
+
ne10, ne11, ne12, ne13, nb00, nb01,
|
| 398 |
+
nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
|
| 399 |
+
break;
|
| 400 |
+
case GGML_TYPE_IQ4_NL:
|
| 401 |
+
set_rows_sycl_q<TIn, TIdx, block_iq4_nl, QK4_NL, cpy_blck_f32_iq4_nl>(
|
| 402 |
+
src0_d, src1_d, (block_iq4_nl *) dst->data, ne00, ne01, ne02, ne03,
|
| 403 |
+
ne10, ne11, ne12, ne13, nb00, nb01,
|
| 404 |
+
nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
|
| 405 |
+
break;
|
| 406 |
+
case GGML_TYPE_MXFP4:
|
| 407 |
+
set_rows_sycl_q<TIn, TIdx, block_mxfp4, QK_MXFP4, cpy_blck_f32_mxfp4>(
|
| 408 |
+
src0_d, src1_d, (block_mxfp4 *) dst->data, ne00, ne01, ne02, ne03,
|
| 409 |
+
ne10, ne11, ne12, ne13, nb00, nb01,
|
| 410 |
+
nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
|
| 411 |
+
break;
|
| 412 |
+
case GGML_TYPE_NVFP4:
|
| 413 |
+
set_rows_sycl_q<TIn, TIdx, block_nvfp4, QK_NVFP4, cpy_blck_f32_nvfp4>(
|
| 414 |
+
src0_d, src1_d, (block_nvfp4 *) dst->data, ne00, ne01, ne02, ne03,
|
| 415 |
+
ne10, ne11, ne12, ne13, nb00, nb01,
|
| 416 |
+
nb02, nb03, nb10, nb11, nb12, nb13, nb1, nb2, nb3, stream);
|
| 417 |
+
break;
|
| 418 |
+
case GGML_TYPE_Q2_K:
|
| 419 |
+
set_rows_sycl_qk_host<TIn, TIdx, block_q2_K, QK_K, quantize_row_q2_K_ref>(
|
| 420 |
+
src0, src1, dst,
|
| 421 |
+
ne00, ne01, ne02, ne03,
|
| 422 |
+
ne11, ne12,
|
| 423 |
+
nb01, nb02, nb03,
|
| 424 |
+
nb10, nb11, nb12,
|
| 425 |
+
nb1, nb2, nb3,
|
| 426 |
+
stream);
|
| 427 |
+
break;
|
| 428 |
+
case GGML_TYPE_Q3_K:
|
| 429 |
+
set_rows_sycl_qk_host<TIn, TIdx, block_q3_K, QK_K, quantize_row_q3_K_ref>(
|
| 430 |
+
src0, src1, dst,
|
| 431 |
+
ne00, ne01, ne02, ne03,
|
| 432 |
+
ne11, ne12,
|
| 433 |
+
nb01, nb02, nb03,
|
| 434 |
+
nb10, nb11, nb12,
|
| 435 |
+
nb1, nb2, nb3,
|
| 436 |
+
stream);
|
| 437 |
+
break;
|
| 438 |
+
case GGML_TYPE_Q4_K:
|
| 439 |
+
set_rows_sycl_qk_host<TIn, TIdx, block_q4_K, QK_K, quantize_row_q4_K_ref>(
|
| 440 |
+
src0, src1, dst,
|
| 441 |
+
ne00, ne01, ne02, ne03,
|
| 442 |
+
ne11, ne12,
|
| 443 |
+
nb01, nb02, nb03,
|
| 444 |
+
nb10, nb11, nb12,
|
| 445 |
+
nb1, nb2, nb3,
|
| 446 |
+
stream);
|
| 447 |
+
break;
|
| 448 |
+
case GGML_TYPE_Q5_K:
|
| 449 |
+
set_rows_sycl_qk_host<TIn, TIdx, block_q5_K, QK_K, quantize_row_q5_K_ref>(
|
| 450 |
+
src0, src1, dst,
|
| 451 |
+
ne00, ne01, ne02, ne03,
|
| 452 |
+
ne11, ne12,
|
| 453 |
+
nb01, nb02, nb03,
|
| 454 |
+
nb10, nb11, nb12,
|
| 455 |
+
nb1, nb2, nb3,
|
| 456 |
+
stream);
|
| 457 |
+
break;
|
| 458 |
+
case GGML_TYPE_Q6_K:
|
| 459 |
+
set_rows_sycl_qk_host<TIn, TIdx, block_q6_K, QK_K, quantize_row_q6_K_ref>(
|
| 460 |
+
src0, src1, dst,
|
| 461 |
+
ne00, ne01, ne02, ne03,
|
| 462 |
+
ne11, ne12,
|
| 463 |
+
nb01, nb02, nb03,
|
| 464 |
+
nb10, nb11, nb12,
|
| 465 |
+
nb1, nb2, nb3,
|
| 466 |
+
stream);
|
| 467 |
+
break;
|
| 468 |
+
case GGML_TYPE_IQ2_XXS:
|
| 469 |
+
set_rows_sycl_iq_host<TIn, TIdx, block_iq2_xxs, QK_K, quantize_iq2_xxs>(
|
| 470 |
+
src0, src1, dst,
|
| 471 |
+
ne00, ne01, ne02, ne03,
|
| 472 |
+
ne11, ne12,
|
| 473 |
+
nb01, nb02, nb03,
|
| 474 |
+
nb10, nb11, nb12,
|
| 475 |
+
nb1, nb2, nb3,
|
| 476 |
+
stream);
|
| 477 |
+
break;
|
| 478 |
+
case GGML_TYPE_IQ2_XS:
|
| 479 |
+
set_rows_sycl_iq_host<TIn, TIdx, block_iq2_xs, QK_K, quantize_iq2_xs>(
|
| 480 |
+
src0, src1, dst,
|
| 481 |
+
ne00, ne01, ne02, ne03,
|
| 482 |
+
ne11, ne12,
|
| 483 |
+
nb01, nb02, nb03,
|
| 484 |
+
nb10, nb11, nb12,
|
| 485 |
+
nb1, nb2, nb3,
|
| 486 |
+
stream);
|
| 487 |
+
break;
|
| 488 |
+
case GGML_TYPE_IQ2_S:
|
| 489 |
+
set_rows_sycl_iq_host<TIn, TIdx, block_iq2_s, QK_K, quantize_iq2_s>(
|
| 490 |
+
src0, src1, dst,
|
| 491 |
+
ne00, ne01, ne02, ne03,
|
| 492 |
+
ne11, ne12,
|
| 493 |
+
nb01, nb02, nb03,
|
| 494 |
+
nb10, nb11, nb12,
|
| 495 |
+
nb1, nb2, nb3,
|
| 496 |
+
stream);
|
| 497 |
+
break;
|
| 498 |
+
case GGML_TYPE_IQ3_XXS:
|
| 499 |
+
set_rows_sycl_qk_host<TIn, TIdx, block_iq3_xxs, QK_K, quantize_row_iq3_xxs_ref>(
|
| 500 |
+
src0, src1, dst,
|
| 501 |
+
ne00, ne01, ne02, ne03,
|
| 502 |
+
ne11, ne12,
|
| 503 |
+
nb01, nb02, nb03,
|
| 504 |
+
nb10, nb11, nb12,
|
| 505 |
+
nb1, nb2, nb3,
|
| 506 |
+
stream);
|
| 507 |
+
break;
|
| 508 |
+
case GGML_TYPE_IQ3_S:
|
| 509 |
+
set_rows_sycl_qk_host<TIn, TIdx, block_iq3_s, QK_K, quantize_row_iq3_s_ref>(
|
| 510 |
+
src0, src1, dst,
|
| 511 |
+
ne00, ne01, ne02, ne03,
|
| 512 |
+
ne11, ne12,
|
| 513 |
+
nb01, nb02, nb03,
|
| 514 |
+
nb10, nb11, nb12,
|
| 515 |
+
nb1, nb2, nb3,
|
| 516 |
+
stream);
|
| 517 |
+
break;
|
| 518 |
+
case GGML_TYPE_IQ1_S:
|
| 519 |
+
set_rows_sycl_iq_host<TIn, TIdx, block_iq1_s, QK_K, quantize_iq1_s>(
|
| 520 |
+
src0, src1, dst,
|
| 521 |
+
ne00, ne01, ne02, ne03,
|
| 522 |
+
ne11, ne12,
|
| 523 |
+
nb01, nb02, nb03,
|
| 524 |
+
nb10, nb11, nb12,
|
| 525 |
+
nb1, nb2, nb3,
|
| 526 |
+
stream);
|
| 527 |
+
break;
|
| 528 |
+
case GGML_TYPE_IQ1_M:
|
| 529 |
+
set_rows_sycl_iq_host<TIn, TIdx, block_iq1_m, QK_K, quantize_iq1_m>(
|
| 530 |
+
src0, src1, dst,
|
| 531 |
+
ne00, ne01, ne02, ne03,
|
| 532 |
+
ne11, ne12,
|
| 533 |
+
nb01, nb02, nb03,
|
| 534 |
+
nb10, nb11, nb12,
|
| 535 |
+
nb1, nb2, nb3,
|
| 536 |
+
stream);
|
| 537 |
+
break;
|
| 538 |
+
case GGML_TYPE_IQ4_XS:
|
| 539 |
+
set_rows_sycl_qk_host<TIn, TIdx, block_iq4_xs, QK_K, quantize_row_iq4_xs_ref>(
|
| 540 |
+
src0, src1, dst,
|
| 541 |
+
ne00, ne01, ne02, ne03,
|
| 542 |
+
ne11, ne12,
|
| 543 |
+
nb01, nb02, nb03,
|
| 544 |
+
nb10, nb11, nb12,
|
| 545 |
+
nb1, nb2, nb3,
|
| 546 |
+
stream);
|
| 547 |
+
break;
|
| 548 |
+
default:
|
| 549 |
+
GGML_ABORT("Unsupported tensor type: src0 %s src1 %s dst %s", ggml_type_name(dst->src[0]->type),
|
| 550 |
+
ggml_type_name(dst->src[1]->type), ggml_type_name(dst->type));
|
| 551 |
+
break;
|
| 552 |
+
}
|
| 553 |
+
}
|
| 554 |
+
|
| 555 |
+
void ggml_sycl_op_set_rows(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
|
| 556 |
+
scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
|
| 557 |
+
const ggml_tensor * src0 = dst->src[0];
|
| 558 |
+
const ggml_tensor * src1 = dst->src[1];
|
| 559 |
+
|
| 560 |
+
GGML_ASSERT(dst->src[0]->type == GGML_TYPE_F32 || dst->src[0]->type == GGML_TYPE_F16);
|
| 561 |
+
GGML_ASSERT(dst->src[1]->type == GGML_TYPE_I64 || dst->src[1]->type == GGML_TYPE_I32);
|
| 562 |
+
|
| 563 |
+
// dispatch on the index type (src1) and the source value type (src0)
|
| 564 |
+
if (src0->type == GGML_TYPE_F16) {
|
| 565 |
+
if (src1->type == GGML_TYPE_I64) {
|
| 566 |
+
set_rows_sycl<sycl::half, int64_t>(ctx, src0, src1, dst);
|
| 567 |
+
} else {
|
| 568 |
+
set_rows_sycl<sycl::half, int32_t>(ctx, src0, src1, dst);
|
| 569 |
+
}
|
| 570 |
+
} else {
|
| 571 |
+
if (src1->type == GGML_TYPE_I64) {
|
| 572 |
+
set_rows_sycl<float, int64_t>(ctx, src0, src1, dst);
|
| 573 |
+
} else {
|
| 574 |
+
set_rows_sycl<float, int32_t>(ctx, src0, src1, dst);
|
| 575 |
+
}
|
| 576 |
+
}
|
| 577 |
+
}
|
ggml/src/ggml-sycl/set_rows.hpp
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#ifndef GGML_SYCL_SET_ROWS_HPP
|
| 2 |
+
#define GGML_SYCL_SET_ROWS_HPP
|
| 3 |
+
|
| 4 |
+
#include "common.hpp"
|
| 5 |
+
|
| 6 |
+
void ggml_sycl_op_set_rows(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
|
| 7 |
+
|
| 8 |
+
#endif // GGML_SYCL_SET_ROWS_HPP
|
ggml/src/ggml-sycl/softmax.cpp
ADDED
|
@@ -0,0 +1,425 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#include "softmax.hpp"
|
| 2 |
+
#include <cstdint>
|
| 3 |
+
#include <utility>
|
| 4 |
+
#include <cmath>
|
| 5 |
+
|
| 6 |
+
|
| 7 |
+
template <typename T> static __dpct_inline__ float t2f32(T val) {
|
| 8 |
+
return (float) val;
|
| 9 |
+
}
|
| 10 |
+
|
| 11 |
+
template <> float __dpct_inline__ t2f32<sycl::half>(sycl::half val) {
|
| 12 |
+
return sycl::vec<sycl::half, 1>(val)
|
| 13 |
+
.convert<float, sycl::rounding_mode::automatic>()[0];
|
| 14 |
+
}
|
| 15 |
+
|
| 16 |
+
struct soft_max_params {
|
| 17 |
+
|
| 18 |
+
int64_t nheads;
|
| 19 |
+
uint32_t n_head_log2;
|
| 20 |
+
int64_t ncols;
|
| 21 |
+
int64_t nrows_x;
|
| 22 |
+
int64_t nrows_y;
|
| 23 |
+
int64_t ne00;
|
| 24 |
+
int64_t ne01;
|
| 25 |
+
int64_t ne02;
|
| 26 |
+
int64_t ne03;
|
| 27 |
+
int64_t nb11;
|
| 28 |
+
int64_t nb12;
|
| 29 |
+
int64_t nb13;
|
| 30 |
+
|
| 31 |
+
int64_t ne12;
|
| 32 |
+
int64_t ne13;
|
| 33 |
+
float scale;
|
| 34 |
+
float max_bias;
|
| 35 |
+
float m0;
|
| 36 |
+
float m1;
|
| 37 |
+
};
|
| 38 |
+
|
| 39 |
+
// When ncols_template == 0 the bounds for the loops in this function are not known and can't be unrolled.
|
| 40 |
+
// As we want to keep pragma unroll for all other cases we suppress the clang transformation warning here.
|
| 41 |
+
#ifdef __clang__
|
| 42 |
+
#pragma clang diagnostic push
|
| 43 |
+
#pragma clang diagnostic ignored "-Wpass-failed"
|
| 44 |
+
#endif // __clang__
|
| 45 |
+
template <bool use_shared, int ncols_template, int block_size_template, typename T>
|
| 46 |
+
static void soft_max_f32(const float * x,
|
| 47 |
+
const T * mask,
|
| 48 |
+
const float * sinks,
|
| 49 |
+
float * dst,
|
| 50 |
+
const soft_max_params p,
|
| 51 |
+
uint8_t * dpct_local) {
|
| 52 |
+
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
|
| 53 |
+
const int ncols = ncols_template == 0 ? p.ncols : ncols_template;
|
| 54 |
+
const int block_size = block_size_template == 0
|
| 55 |
+
? item_ct1.get_local_range(2)
|
| 56 |
+
: block_size_template;
|
| 57 |
+
const int nthreads = block_size;
|
| 58 |
+
const int nwarps = nthreads / WARP_SIZE;
|
| 59 |
+
const size_t nreduce = nwarps / WARP_SIZE;
|
| 60 |
+
|
| 61 |
+
const int tid = item_ct1.get_local_id(2);
|
| 62 |
+
|
| 63 |
+
const int64_t i03 = item_ct1.get_group(0);
|
| 64 |
+
const int64_t i02 = item_ct1.get_group(1);
|
| 65 |
+
const int64_t i01 = item_ct1.get_group(2);
|
| 66 |
+
|
| 67 |
+
//TODO: noncontigous inputs/outputs
|
| 68 |
+
const int rowx = item_ct1.get_group(2) +
|
| 69 |
+
item_ct1.get_group(1) * item_ct1.get_group_range(2) +
|
| 70 |
+
item_ct1.get_group(0) * item_ct1.get_group_range(2) *
|
| 71 |
+
item_ct1.get_group_range(1);
|
| 72 |
+
|
| 73 |
+
const int64_t i11 = i01;
|
| 74 |
+
const int64_t i12 = i02 % p.ne12;
|
| 75 |
+
const int64_t i13 = i03 % p.ne13;
|
| 76 |
+
|
| 77 |
+
x += int64_t(rowx)*ncols;
|
| 78 |
+
mask += (i11*p.nb11 + i12*p.nb12 + i13*p.nb13) / sizeof(T) * (mask != nullptr);
|
| 79 |
+
dst += int64_t(rowx)*ncols;
|
| 80 |
+
|
| 81 |
+
const int warp_id = item_ct1.get_local_id(2) / WARP_SIZE;
|
| 82 |
+
const int lane_id = item_ct1.get_local_id(2) % WARP_SIZE;
|
| 83 |
+
|
| 84 |
+
const float slope = get_alibi_slope(p.max_bias, i02, p.n_head_log2, p.m0, p.m1);
|
| 85 |
+
|
| 86 |
+
float * buf_iw = (float *) dpct_local;
|
| 87 |
+
|
| 88 |
+
// shared memory buffer to cache values between iterations:
|
| 89 |
+
float *vals = use_shared ? buf_iw + sycl::max(nwarps, WARP_SIZE) : dst;
|
| 90 |
+
float max_val = sinks ? sinks[i02] : -INFINITY;
|
| 91 |
+
#pragma unroll
|
| 92 |
+
for (int col0 = 0; col0 < ncols; col0 += block_size) {
|
| 93 |
+
const int col = col0 + tid;
|
| 94 |
+
|
| 95 |
+
if (ncols_template == 0 && col >= ncols) {
|
| 96 |
+
break;
|
| 97 |
+
}
|
| 98 |
+
|
| 99 |
+
const float val = x[col]*p.scale + (mask ? slope*t2f32(mask[col]) : 0.0f);
|
| 100 |
+
|
| 101 |
+
vals[col] = val;
|
| 102 |
+
max_val = sycl::max(max_val, val);
|
| 103 |
+
}
|
| 104 |
+
// find the max value in the block
|
| 105 |
+
max_val = warp_reduce_max<WARP_SIZE>(max_val);
|
| 106 |
+
|
| 107 |
+
if (block_size > WARP_SIZE) {
|
| 108 |
+
if (lane_id == 0) {
|
| 109 |
+
buf_iw[warp_id] = max_val;
|
| 110 |
+
}
|
| 111 |
+
item_ct1.barrier();
|
| 112 |
+
|
| 113 |
+
max_val = -INFINITY;
|
| 114 |
+
for (int i = lane_id; i < nwarps; i += WARP_SIZE) {
|
| 115 |
+
max_val = sycl::max(max_val, buf_iw[i]);
|
| 116 |
+
}
|
| 117 |
+
max_val = warp_reduce_max<WARP_SIZE>(max_val);
|
| 118 |
+
}
|
| 119 |
+
float tmp = 0.0f; // partial sum
|
| 120 |
+
|
| 121 |
+
#pragma unroll
|
| 122 |
+
for (int col0 = 0; col0 < ncols; col0 += block_size) {
|
| 123 |
+
const int col = col0 + tid;
|
| 124 |
+
|
| 125 |
+
if (ncols_template == 0 && col >= ncols) {
|
| 126 |
+
break;
|
| 127 |
+
}
|
| 128 |
+
|
| 129 |
+
const float val = sycl::native::exp(sycl::max(vals[col] - max_val, -80.0f));
|
| 130 |
+
tmp += val;
|
| 131 |
+
vals[col] = val;
|
| 132 |
+
}
|
| 133 |
+
// find the sum of exps in the block
|
| 134 |
+
tmp = warp_reduce_sum<WARP_SIZE>(tmp);
|
| 135 |
+
if (block_size > WARP_SIZE) {
|
| 136 |
+
item_ct1.barrier();
|
| 137 |
+
if (warp_id == 0) {
|
| 138 |
+
buf_iw[lane_id] = 0.0f;
|
| 139 |
+
for (size_t i = 1; i < nreduce; i += 1) {
|
| 140 |
+
buf_iw[lane_id + i * WARP_SIZE] = 0.f;
|
| 141 |
+
}
|
| 142 |
+
}
|
| 143 |
+
item_ct1.barrier();
|
| 144 |
+
|
| 145 |
+
if (lane_id == 0) {
|
| 146 |
+
buf_iw[warp_id] = tmp;
|
| 147 |
+
}
|
| 148 |
+
item_ct1.barrier();
|
| 149 |
+
|
| 150 |
+
tmp = buf_iw[lane_id];
|
| 151 |
+
for (size_t i = 1; i < nreduce; i += 1) {
|
| 152 |
+
tmp += buf_iw[lane_id + i * WARP_SIZE];
|
| 153 |
+
}
|
| 154 |
+
tmp = warp_reduce_sum<WARP_SIZE>(tmp);
|
| 155 |
+
}
|
| 156 |
+
if (sinks) {
|
| 157 |
+
tmp += sycl::native::exp(sycl::max(sinks[i02] - max_val, -80.0f));
|
| 158 |
+
}
|
| 159 |
+
const float inv_sum = 1.0f / tmp;
|
| 160 |
+
|
| 161 |
+
#pragma unroll
|
| 162 |
+
for (int col0 = 0; col0 < ncols; col0 += block_size) {
|
| 163 |
+
const int col = col0 + tid;
|
| 164 |
+
|
| 165 |
+
if (ncols_template == 0 && col >= ncols) {
|
| 166 |
+
return;
|
| 167 |
+
}
|
| 168 |
+
|
| 169 |
+
dst[col] = vals[col] * inv_sum;
|
| 170 |
+
}
|
| 171 |
+
}
|
| 172 |
+
#ifdef __clang__
|
| 173 |
+
#pragma clang diagnostic pop
|
| 174 |
+
#endif // __clang__
|
| 175 |
+
|
| 176 |
+
static void soft_max_back_f32(const float *grad, const float *dstf, float *dst,
|
| 177 |
+
const int ncols, const float scale) {
|
| 178 |
+
auto item_ct1 = sycl::ext::oneapi::this_work_item::get_nd_item<3>();
|
| 179 |
+
const int tid = item_ct1.get_local_id(2);
|
| 180 |
+
const int rowx = item_ct1.get_group(2);
|
| 181 |
+
|
| 182 |
+
grad += int64_t(rowx)*ncols;
|
| 183 |
+
dstf += int64_t(rowx)*ncols;
|
| 184 |
+
dst += int64_t(rowx)*ncols;
|
| 185 |
+
|
| 186 |
+
float dgf_dot = 0.0f; // dot product of dst from forward pass and gradients
|
| 187 |
+
|
| 188 |
+
for (int col = tid; col < ncols; col += WARP_SIZE) {
|
| 189 |
+
dgf_dot += dstf[col]*grad[col];
|
| 190 |
+
}
|
| 191 |
+
|
| 192 |
+
dgf_dot = warp_reduce_sum<WARP_SIZE>(dgf_dot);
|
| 193 |
+
|
| 194 |
+
for (int col = tid; col < ncols; col += WARP_SIZE) {
|
| 195 |
+
dst[col] = scale * (grad[col] - dgf_dot) * dstf[col];
|
| 196 |
+
}
|
| 197 |
+
}
|
| 198 |
+
|
| 199 |
+
template <int... Ns, typename T>
|
| 200 |
+
static void launch_soft_max_kernels(const float * x,
|
| 201 |
+
const T * mask,
|
| 202 |
+
const float * sinks,
|
| 203 |
+
float * dst,
|
| 204 |
+
const soft_max_params & p,
|
| 205 |
+
dpct::queue_ptr stream,
|
| 206 |
+
dpct::dim3 block_dims,
|
| 207 |
+
dpct::dim3 block_nums,
|
| 208 |
+
size_t nbytes_shared)
|
| 209 |
+
{
|
| 210 |
+
auto launch_kernel = [=](auto I) -> bool {
|
| 211 |
+
constexpr int ncols = decltype(I)::value;
|
| 212 |
+
constexpr int block = (ncols > 1024 ? 1024 : ncols);
|
| 213 |
+
if (p.ncols == ncols) {
|
| 214 |
+
stream->submit([&](sycl::handler &cgh) {
|
| 215 |
+
sycl::local_accessor<uint8_t, 1> dpct_local_acc_ct1(
|
| 216 |
+
sycl::range<1>(nbytes_shared), cgh);
|
| 217 |
+
|
| 218 |
+
cgh.parallel_for(
|
| 219 |
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
| 220 |
+
[=](sycl::nd_item<3> item_ct1) [[sycl::reqd_sub_group_size(
|
| 221 |
+
WARP_SIZE)]] {
|
| 222 |
+
soft_max_f32<true, ncols, block>(
|
| 223 |
+
x, mask, sinks, dst, p,
|
| 224 |
+
dpct_local_acc_ct1
|
| 225 |
+
.get_multi_ptr<sycl::access::decorated::no>()
|
| 226 |
+
.get());
|
| 227 |
+
GGML_UNUSED(item_ct1);
|
| 228 |
+
});
|
| 229 |
+
});
|
| 230 |
+
return true;
|
| 231 |
+
}
|
| 232 |
+
return false;
|
| 233 |
+
};
|
| 234 |
+
|
| 235 |
+
// unary fold over launch_kernel
|
| 236 |
+
if ((launch_kernel(std::integral_constant<int, Ns>{}) || ...)) {
|
| 237 |
+
return;
|
| 238 |
+
}
|
| 239 |
+
|
| 240 |
+
stream->submit([&](sycl::handler &cgh) {
|
| 241 |
+
sycl::local_accessor<uint8_t, 1> dpct_local_acc_ct1(
|
| 242 |
+
sycl::range<1>(nbytes_shared), cgh);
|
| 243 |
+
|
| 244 |
+
cgh.parallel_for(
|
| 245 |
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
| 246 |
+
[=](sycl::nd_item<3> item_ct1)
|
| 247 |
+
[[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
| 248 |
+
soft_max_f32<true, 0, 0>(
|
| 249 |
+
x, mask, sinks, dst, p,
|
| 250 |
+
dpct_local_acc_ct1
|
| 251 |
+
.get_multi_ptr<sycl::access::decorated::no>()
|
| 252 |
+
.get());
|
| 253 |
+
GGML_UNUSED(item_ct1);
|
| 254 |
+
});
|
| 255 |
+
});
|
| 256 |
+
}
|
| 257 |
+
|
| 258 |
+
template <typename T>
|
| 259 |
+
static void soft_max_f32_sycl(const float *x, const T *mask,
|
| 260 |
+
const float *sinks, float *dst,
|
| 261 |
+
const soft_max_params ¶ms,
|
| 262 |
+
dpct::queue_ptr stream, int device) {
|
| 263 |
+
int nth = WARP_SIZE;
|
| 264 |
+
int max_block_size = ggml_sycl_info().max_work_group_sizes[device];
|
| 265 |
+
const int64_t ncols_x = params.ncols;
|
| 266 |
+
|
| 267 |
+
while (nth < ncols_x && nth < max_block_size) nth *= 2;
|
| 268 |
+
if (nth>max_block_size) nth = max_block_size;
|
| 269 |
+
|
| 270 |
+
const dpct::dim3 block_dims(nth, 1, 1);
|
| 271 |
+
const dpct::dim3 block_nums(params.ne01, params.ne02, params.ne03);
|
| 272 |
+
const size_t nbytes_shared =
|
| 273 |
+
(GGML_PAD(ncols_x, WARP_SIZE) + WARP_SIZE) * sizeof(float);
|
| 274 |
+
|
| 275 |
+
const int id = get_current_device_id();
|
| 276 |
+
const size_t smpbo = ggml_sycl_info().devices[id].smpbo;
|
| 277 |
+
|
| 278 |
+
if (nbytes_shared <= smpbo && ncols_x <= max_block_size) {
|
| 279 |
+
launch_soft_max_kernels<32, 64, 128, 256, 512, 1024, 2048, 4096>(
|
| 280 |
+
x, mask, sinks, dst, params, stream, block_dims, block_nums,
|
| 281 |
+
nbytes_shared);
|
| 282 |
+
} else {
|
| 283 |
+
const size_t nbytes_shared_low = WARP_SIZE * sizeof(float);
|
| 284 |
+
|
| 285 |
+
stream->submit([&](sycl::handler &cgh) {
|
| 286 |
+
sycl::local_accessor<uint8_t, 1> dpct_local_acc_ct1(
|
| 287 |
+
sycl::range<1>(nbytes_shared_low), cgh);
|
| 288 |
+
|
| 289 |
+
cgh.parallel_for(
|
| 290 |
+
sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
| 291 |
+
[=](sycl::nd_item<3> item_ct1)
|
| 292 |
+
[[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
| 293 |
+
soft_max_f32<false, 0, 0>(
|
| 294 |
+
x, mask, sinks, dst, params,
|
| 295 |
+
dpct_local_acc_ct1
|
| 296 |
+
.get_multi_ptr<sycl::access::decorated::no>()
|
| 297 |
+
.get());
|
| 298 |
+
GGML_UNUSED(item_ct1);
|
| 299 |
+
});
|
| 300 |
+
});
|
| 301 |
+
}
|
| 302 |
+
}
|
| 303 |
+
|
| 304 |
+
static void soft_max_back_f32_sycl(const float * grad,
|
| 305 |
+
const float * dstf,
|
| 306 |
+
float * dst,
|
| 307 |
+
const int ncols,
|
| 308 |
+
const int nrows,
|
| 309 |
+
const float scale,
|
| 310 |
+
dpct::queue_ptr stream) {
|
| 311 |
+
const dpct::dim3 block_dims(WARP_SIZE, 1, 1);
|
| 312 |
+
const dpct::dim3 block_nums(nrows, 1, 1);
|
| 313 |
+
|
| 314 |
+
stream->parallel_for(sycl::nd_range<3>(block_nums * block_dims, block_dims),
|
| 315 |
+
[=](sycl::nd_item<3> item_ct1) {
|
| 316 |
+
soft_max_back_f32(grad, dstf, dst, ncols, scale);
|
| 317 |
+
GGML_UNUSED(item_ct1);
|
| 318 |
+
});
|
| 319 |
+
}
|
| 320 |
+
|
| 321 |
+
void ggml_sycl_op_soft_max(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
|
| 322 |
+
scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
|
| 323 |
+
|
| 324 |
+
const ggml_tensor * src0 = dst->src[0];
|
| 325 |
+
const ggml_tensor * src1 = dst->src[1];
|
| 326 |
+
const ggml_tensor * src2 = dst->src[2];
|
| 327 |
+
|
| 328 |
+
const float * src0_d = (const float *) src0->data;
|
| 329 |
+
const void * src1_d = src1 ? (const void *) src1->data : nullptr;
|
| 330 |
+
const void * src2_d = src2 ? (const void *) src2->data : nullptr;
|
| 331 |
+
float * dst_d = (float *) dst->data;
|
| 332 |
+
|
| 333 |
+
dpct::queue_ptr stream = ctx.stream();
|
| 334 |
+
|
| 335 |
+
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
| 336 |
+
GGML_ASSERT( dst->type == GGML_TYPE_F32);
|
| 337 |
+
|
| 338 |
+
// src1 contains mask and it is optional
|
| 339 |
+
GGML_ASSERT(!src1 || src1->type == GGML_TYPE_F16 || src1->type == GGML_TYPE_F32);
|
| 340 |
+
|
| 341 |
+
const int64_t nrows_x = ggml_nrows(src0);
|
| 342 |
+
const int64_t nrows_y = src0->ne[1];
|
| 343 |
+
|
| 344 |
+
const int64_t ne00 = src0->ne[0];
|
| 345 |
+
|
| 346 |
+
float scale = 1.0f;
|
| 347 |
+
float max_bias = 0.0f;
|
| 348 |
+
|
| 349 |
+
memcpy(&scale, (const float *) dst->op_params + 0, sizeof(float));
|
| 350 |
+
memcpy(&max_bias, (const float *) dst->op_params + 1, sizeof(float));
|
| 351 |
+
|
| 352 |
+
const bool use_f16 = (src1 && src1->type == GGML_TYPE_F16);
|
| 353 |
+
|
| 354 |
+
const int64_t nb11 = src1 ? src1->nb[1] : 1;
|
| 355 |
+
const int64_t nb12 = src1 ? src1->nb[2] : 1;
|
| 356 |
+
const int64_t nb13 = src1 ? src1->nb[3] : 1;
|
| 357 |
+
|
| 358 |
+
const int64_t ne12 = src1 ? src1->ne[2] : 1;
|
| 359 |
+
const int64_t ne13 = src1 ? src1->ne[3] : 1;
|
| 360 |
+
|
| 361 |
+
const uint32_t n_head = src0->ne[2];
|
| 362 |
+
const uint32_t n_head_log2 = 1u << (uint32_t) floorf(log2f((float) n_head));
|
| 363 |
+
|
| 364 |
+
const float m0 = powf(2.0f, -(max_bias ) / n_head_log2);
|
| 365 |
+
const float m1 = powf(2.0f, -(max_bias / 2.0f) / n_head_log2);
|
| 366 |
+
|
| 367 |
+
|
| 368 |
+
soft_max_params params = {};
|
| 369 |
+
params.nheads = src0->ne[2];
|
| 370 |
+
params.n_head_log2 = n_head_log2;
|
| 371 |
+
params.ncols = ne00;
|
| 372 |
+
params.nrows_x = nrows_x;
|
| 373 |
+
params.nrows_y = nrows_y;
|
| 374 |
+
params.ne00 = src0->ne[0];
|
| 375 |
+
params.ne01 = src0->ne[1];
|
| 376 |
+
params.ne02 = src0->ne[2];
|
| 377 |
+
params.ne03 = src0->ne[3];
|
| 378 |
+
params.nb11 = nb11;
|
| 379 |
+
params.nb12 = nb12;
|
| 380 |
+
params.nb13 = nb13;
|
| 381 |
+
params.ne12 = ne12;
|
| 382 |
+
params.ne13 = ne13;
|
| 383 |
+
params.scale = scale;
|
| 384 |
+
params.max_bias = max_bias;
|
| 385 |
+
params.m0 = m0;
|
| 386 |
+
params.m1 = m1;
|
| 387 |
+
|
| 388 |
+
if (use_f16) {
|
| 389 |
+
soft_max_f32_sycl(src0_d, (const sycl::half *)src1_d,
|
| 390 |
+
(const float *)src2_d, dst_d, params, stream,
|
| 391 |
+
ctx.device);
|
| 392 |
+
} else {
|
| 393 |
+
soft_max_f32_sycl(src0_d, (const float *)src1_d, (const float *)src2_d,
|
| 394 |
+
dst_d, params, stream, ctx.device);
|
| 395 |
+
}
|
| 396 |
+
}
|
| 397 |
+
|
| 398 |
+
void ggml_sycl_op_soft_max_back(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
|
| 399 |
+
scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
|
| 400 |
+
const ggml_tensor * src0 = dst->src[0]; // grad
|
| 401 |
+
const ggml_tensor * src1 = dst->src[1]; // forward pass output
|
| 402 |
+
|
| 403 |
+
const float * src0_d = (const float *) src0->data;
|
| 404 |
+
const float * src1_d = (const float *) src1->data;
|
| 405 |
+
float * dst_d = (float *) dst->data;
|
| 406 |
+
|
| 407 |
+
dpct::queue_ptr stream = ctx.stream();
|
| 408 |
+
|
| 409 |
+
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
| 410 |
+
GGML_ASSERT(src1->type == GGML_TYPE_F32);
|
| 411 |
+
GGML_ASSERT( dst->type == GGML_TYPE_F32);
|
| 412 |
+
|
| 413 |
+
const int64_t ncols = src0->ne[0];
|
| 414 |
+
const int64_t nrows = ggml_nrows(src0);
|
| 415 |
+
|
| 416 |
+
float scale = 1.0f;
|
| 417 |
+
float max_bias = 0.0f;
|
| 418 |
+
|
| 419 |
+
memcpy(&scale, (const float *) dst->op_params + 0, sizeof(float));
|
| 420 |
+
memcpy(&max_bias, (const float *) dst->op_params + 1, sizeof(float));
|
| 421 |
+
|
| 422 |
+
GGML_ASSERT(max_bias == 0.0f);
|
| 423 |
+
|
| 424 |
+
soft_max_back_f32_sycl(src0_d, src1_d, dst_d, ncols, nrows, scale, stream);
|
| 425 |
+
}
|
ggml/src/ggml-sycl/softmax.hpp
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
//
|
| 2 |
+
// MIT license
|
| 3 |
+
// Copyright (C) 2024 Intel Corporation
|
| 4 |
+
// SPDX-License-Identifier: MIT
|
| 5 |
+
//
|
| 6 |
+
|
| 7 |
+
//
|
| 8 |
+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
| 9 |
+
// See https://llvm.org/LICENSE.txt for license information.
|
| 10 |
+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
| 11 |
+
//
|
| 12 |
+
|
| 13 |
+
#ifndef GGML_SYCL_SOFTMAX_HPP
|
| 14 |
+
#define GGML_SYCL_SOFTMAX_HPP
|
| 15 |
+
|
| 16 |
+
#include "common.hpp"
|
| 17 |
+
|
| 18 |
+
#define SYCL_SOFT_MAX_BLOCK_SIZE 1024
|
| 19 |
+
|
| 20 |
+
void ggml_sycl_op_soft_max(ggml_backend_sycl_context &ctx, ggml_tensor *dst);
|
| 21 |
+
|
| 22 |
+
void ggml_sycl_op_soft_max_back(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
|
| 23 |
+
|
| 24 |
+
#endif // GGML_SYCL_SOFTMAX_HPP
|
ggml/src/ggml-sycl/solve_tri.cpp
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#include "solve_tri.hpp"
|
| 2 |
+
#include "common.hpp"
|
| 3 |
+
#include <oneapi/mkl/blas.hpp>
|
| 4 |
+
|
| 5 |
+
template <int n_template, int k_template>
|
| 6 |
+
static void solve_tri_f32_fast(const float * __restrict__ A,
|
| 7 |
+
const float * __restrict__ B,
|
| 8 |
+
float * __restrict__ X,
|
| 9 |
+
const int64_t ne02, [[maybe_unused]] const int64_t ne03,
|
| 10 |
+
const int64_t nb02, const int64_t nb03,
|
| 11 |
+
const int64_t nb12, const int64_t nb13,
|
| 12 |
+
const int64_t nb2, const int64_t nb3,
|
| 13 |
+
const int n_arg, const int k_arg,
|
| 14 |
+
const sycl::nd_item<2> & item, float * sA) {
|
| 15 |
+
|
| 16 |
+
const int n = n_template == 0 ? n_arg : n_template;
|
| 17 |
+
const int k = k_template == 0 ? k_arg : k_template;
|
| 18 |
+
|
| 19 |
+
const int batch_idx = item.get_group(1);
|
| 20 |
+
const int lane = item.get_local_id(1) % WARP_SIZE;
|
| 21 |
+
const int col_idx = item.get_local_id(0);
|
| 22 |
+
|
| 23 |
+
if (col_idx >= k) {
|
| 24 |
+
return;
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
const int64_t i03 = batch_idx / ne02;
|
| 28 |
+
const int64_t i02 = batch_idx % ne02;
|
| 29 |
+
|
| 30 |
+
const float * A_batch = (const float *) ((const char *) A + i02 * nb02 + i03 * nb03);
|
| 31 |
+
const float * B_batch = (const float *) ((const char *) B + i02 * nb12 + i03 * nb13);
|
| 32 |
+
float * X_batch = (float *) ((char *) X + i02 * nb2 + i03 * nb3);
|
| 33 |
+
|
| 34 |
+
const int offset = item.get_local_id(1) + item.get_local_id(0) * item.get_local_range(1);
|
| 35 |
+
|
| 36 |
+
#pragma unroll
|
| 37 |
+
for (int i = 0; i < n * n; i += k * WARP_SIZE) {
|
| 38 |
+
const int i0 = i + offset;
|
| 39 |
+
if (i0 < n * n) {
|
| 40 |
+
sA[i0] = A_batch[i0];
|
| 41 |
+
}
|
| 42 |
+
}
|
| 43 |
+
|
| 44 |
+
item.barrier(sycl::access::fence_space::local_space);
|
| 45 |
+
|
| 46 |
+
float x_low = (lane < n) ? B_batch[lane * k + col_idx] : 0.0f;
|
| 47 |
+
float x_high = (WARP_SIZE + lane < n) ? B_batch[(WARP_SIZE + lane) * k + col_idx] : 0.0f;
|
| 48 |
+
|
| 49 |
+
const int half = WARP_SIZE;
|
| 50 |
+
const int nrows_low = (n < half) ? n : half;
|
| 51 |
+
|
| 52 |
+
#pragma unroll
|
| 53 |
+
for (int row = 0; row < nrows_low; ++row) {
|
| 54 |
+
float sum = 0.0f;
|
| 55 |
+
if (lane < row) {
|
| 56 |
+
sum += sA[row * n + lane] * x_low;
|
| 57 |
+
}
|
| 58 |
+
sum = warp_reduce_sum<WARP_SIZE>(sum);
|
| 59 |
+
if (lane == row) {
|
| 60 |
+
x_low = (x_low - sum) / sA[row * n + row];
|
| 61 |
+
}
|
| 62 |
+
}
|
| 63 |
+
|
| 64 |
+
#pragma unroll
|
| 65 |
+
for (int row = half; row < n; ++row) {
|
| 66 |
+
float sum = sA[row * n + lane] * x_low;
|
| 67 |
+
const int j = half + lane;
|
| 68 |
+
if (j < row) {
|
| 69 |
+
sum += sA[row * n + j] * x_high;
|
| 70 |
+
}
|
| 71 |
+
sum = warp_reduce_sum<WARP_SIZE>(sum);
|
| 72 |
+
if (lane == row - half) {
|
| 73 |
+
x_high = (x_high - sum) / sA[row * n + row];
|
| 74 |
+
}
|
| 75 |
+
}
|
| 76 |
+
|
| 77 |
+
#pragma unroll
|
| 78 |
+
for (int rr = 0; rr < 2; ++rr) {
|
| 79 |
+
const int row = rr * WARP_SIZE + lane;
|
| 80 |
+
if (row < n) {
|
| 81 |
+
const float val = (row < half) ? x_low : x_high;
|
| 82 |
+
X_batch[row * k + col_idx] = val;
|
| 83 |
+
}
|
| 84 |
+
}
|
| 85 |
+
}
|
| 86 |
+
|
| 87 |
+
static void solve_tri_f32_mkl(dpct::queue_ptr stream,
|
| 88 |
+
const float * A, float * X,
|
| 89 |
+
int n, int k,
|
| 90 |
+
int64_t ne02, [[maybe_unused]] int64_t ne03,
|
| 91 |
+
int64_t nb02, [[maybe_unused]] int64_t nb03,
|
| 92 |
+
int64_t nb2, [[maybe_unused]] int64_t nb3) {
|
| 93 |
+
const float alpha = 1.0f;
|
| 94 |
+
const int64_t total_batches = ne02 * ne03;
|
| 95 |
+
if (total_batches == 0) {
|
| 96 |
+
return;
|
| 97 |
+
}
|
| 98 |
+
|
| 99 |
+
const int64_t stride_a = nb02 / sizeof(float);
|
| 100 |
+
const int64_t stride_x = nb2 / sizeof(float);
|
| 101 |
+
|
| 102 |
+
oneapi::mkl::blas::trsm_batch(
|
| 103 |
+
*stream,
|
| 104 |
+
oneapi::mkl::side::right,
|
| 105 |
+
oneapi::mkl::uplo::upper,
|
| 106 |
+
oneapi::mkl::transpose::nontrans,
|
| 107 |
+
oneapi::mkl::diag::nonunit,
|
| 108 |
+
k, n, alpha,
|
| 109 |
+
A, n, stride_a,
|
| 110 |
+
X, k, stride_x,
|
| 111 |
+
total_batches);
|
| 112 |
+
}
|
| 113 |
+
|
| 114 |
+
inline void ggml_sycl_op_solve_tri(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
|
| 115 |
+
const ggml_tensor * src0 = dst->src[0];
|
| 116 |
+
const ggml_tensor * src1 = dst->src[1];
|
| 117 |
+
|
| 118 |
+
GGML_ASSERT(ggml_is_contiguous(src0));
|
| 119 |
+
GGML_ASSERT(ggml_is_contiguous(src1));
|
| 120 |
+
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
| 121 |
+
|
| 122 |
+
dpct::queue_ptr stream = ctx.stream();
|
| 123 |
+
SYCL_CHECK(ggml_sycl_set_device(ctx.device));
|
| 124 |
+
|
| 125 |
+
const int n = src0->ne[0];
|
| 126 |
+
const int k = src1->ne[0];
|
| 127 |
+
const int64_t ne02 = src0->ne[2];
|
| 128 |
+
const int64_t ne03 = src0->ne[3];
|
| 129 |
+
|
| 130 |
+
GGML_ASSERT(n <= SYCL_SOLVE_TRI_MAX_N && k <= SYCL_SOLVE_TRI_MAX_K);
|
| 131 |
+
|
| 132 |
+
const float * A_d = static_cast<const float *>(src0->data);
|
| 133 |
+
const float * B_d = static_cast<const float *>(src1->data);
|
| 134 |
+
float * X_d = static_cast<float *>(dst->data);
|
| 135 |
+
|
| 136 |
+
if (X_d != B_d) {
|
| 137 |
+
const int64_t total_elements = (int64_t)n * k * ne02 * ne03;
|
| 138 |
+
stream->memcpy(X_d, B_d, total_elements * sizeof(float));
|
| 139 |
+
}
|
| 140 |
+
|
| 141 |
+
const int64_t nb02 = src0->nb[2];
|
| 142 |
+
const int64_t nb03 = src0->nb[3];
|
| 143 |
+
const int64_t nb12 = src1->nb[2];
|
| 144 |
+
const int64_t nb13 = src1->nb[3];
|
| 145 |
+
const int64_t nb2 = dst->nb[2];
|
| 146 |
+
const int64_t nb3 = dst->nb[3];
|
| 147 |
+
|
| 148 |
+
const int64_t total_batches = ne02 * ne03;
|
| 149 |
+
|
| 150 |
+
if (n <= 2 * WARP_SIZE && k <= 32) {
|
| 151 |
+
const int smem_size = 2 * WARP_SIZE * 2 * WARP_SIZE;
|
| 152 |
+
const sycl::range<2> grid(1, total_batches);
|
| 153 |
+
const sycl::range<2> block(k, WARP_SIZE);
|
| 154 |
+
stream->submit([&](sycl::handler & cgh) {
|
| 155 |
+
sycl::local_accessor<float, 1> smem_acc(sycl::range<1>(smem_size), cgh);
|
| 156 |
+
cgh.parallel_for(
|
| 157 |
+
sycl::nd_range<2>(grid * block, block),
|
| 158 |
+
[=](sycl::nd_item<2> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
| 159 |
+
solve_tri_f32_fast<0, 0>(A_d, B_d, X_d, ne02, ne03,
|
| 160 |
+
nb02, nb03, nb12, nb13, nb2, nb3,
|
| 161 |
+
n, k, item, get_pointer(smem_acc));
|
| 162 |
+
});
|
| 163 |
+
});
|
| 164 |
+
} else {
|
| 165 |
+
solve_tri_f32_mkl(stream, A_d, X_d, n, k, ne02, ne03, nb02, nb03, nb2, nb3);
|
| 166 |
+
}
|
| 167 |
+
}
|
| 168 |
+
|
| 169 |
+
void ggml_sycl_solve_tri(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
|
| 170 |
+
scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
|
| 171 |
+
ggml_sycl_op_solve_tri(ctx, dst);
|
| 172 |
+
}
|
ggml/src/ggml-sycl/solve_tri.hpp
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#pragma once
|
| 2 |
+
|
| 3 |
+
#include "common.hpp"
|
| 4 |
+
|
| 5 |
+
#define SYCL_SOLVE_TRI_MAX_N 64
|
| 6 |
+
#define SYCL_SOLVE_TRI_MAX_K 64
|
| 7 |
+
|
| 8 |
+
void ggml_sycl_solve_tri(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
|
ggml/src/ggml-sycl/ssm_conv.cpp
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#include "ssm_conv.hpp"
|
| 2 |
+
#include "common.hpp"
|
| 3 |
+
|
| 4 |
+
#include <cstdio>
|
| 5 |
+
|
| 6 |
+
using namespace sycl;
|
| 7 |
+
|
| 8 |
+
static void kernel_ssm_conv(
|
| 9 |
+
queue &q,
|
| 10 |
+
const float *src_data,
|
| 11 |
+
const float *weights,
|
| 12 |
+
float *dst_data,
|
| 13 |
+
int d_conv,
|
| 14 |
+
int d_inner,
|
| 15 |
+
int n_t,
|
| 16 |
+
int n_s,
|
| 17 |
+
int ncs __attribute__((unused)),
|
| 18 |
+
int src_stride_inner,
|
| 19 |
+
int src_stride_seq,
|
| 20 |
+
int dst_stride_token,
|
| 21 |
+
int dst_stride_seq
|
| 22 |
+
) {
|
| 23 |
+
const size_t total_work = static_cast<size_t>(d_inner) * static_cast<size_t>(n_t) * static_cast<size_t>(n_s);
|
| 24 |
+
const size_t work_group_size = 256;
|
| 25 |
+
const size_t num_work_groups = (total_work + work_group_size - 1) / work_group_size;
|
| 26 |
+
|
| 27 |
+
const range<1> global_range(num_work_groups * work_group_size);
|
| 28 |
+
const range<1> local_range(work_group_size);
|
| 29 |
+
|
| 30 |
+
q.submit([&](handler &h) {
|
| 31 |
+
h.parallel_for(
|
| 32 |
+
nd_range<1>(global_range, local_range),
|
| 33 |
+
[=](nd_item<1> item) {
|
| 34 |
+
const size_t idx = item.get_global_id(0);
|
| 35 |
+
if (idx >= total_work) {
|
| 36 |
+
return;
|
| 37 |
+
}
|
| 38 |
+
|
| 39 |
+
// src has the tokens of one channel contiguous, dst has the channels of one
|
| 40 |
+
// token contiguous, so either the loads or the store must be strided. Indexing
|
| 41 |
+
// token-fastest coalesces the d_conv loads, which measured faster except for
|
| 42 |
+
// short, cache-resident rows.
|
| 43 |
+
const int token = static_cast<int>(idx % n_t);
|
| 44 |
+
const int channel = static_cast<int>((idx / n_t) % d_inner);
|
| 45 |
+
const int seq = static_cast<int>(idx / (static_cast<size_t>(n_t) * static_cast<size_t>(d_inner)));
|
| 46 |
+
|
| 47 |
+
const float *s = src_data
|
| 48 |
+
+ static_cast<size_t>(seq) * static_cast<size_t>(src_stride_seq)
|
| 49 |
+
+ static_cast<size_t>(channel) * static_cast<size_t>(src_stride_inner)
|
| 50 |
+
+ static_cast<size_t>(token);
|
| 51 |
+
|
| 52 |
+
const float *c = weights + static_cast<size_t>(channel) * static_cast<size_t>(d_conv);
|
| 53 |
+
|
| 54 |
+
float sumf = 0.0f;
|
| 55 |
+
for (int i0 = 0; i0 < d_conv; ++i0) {
|
| 56 |
+
sumf += s[i0] * c[i0];
|
| 57 |
+
}
|
| 58 |
+
|
| 59 |
+
const size_t dst_idx =
|
| 60 |
+
static_cast<size_t>(seq) * static_cast<size_t>(dst_stride_seq) +
|
| 61 |
+
static_cast<size_t>(token) * static_cast<size_t>(dst_stride_token) +
|
| 62 |
+
static_cast<size_t>(channel);
|
| 63 |
+
|
| 64 |
+
dst_data[dst_idx] = sumf;
|
| 65 |
+
}
|
| 66 |
+
);
|
| 67 |
+
});
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
inline void ggml_sycl_op_ssm_conv(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
|
| 71 |
+
ggml_tensor * src0 = dst->src[0];
|
| 72 |
+
ggml_tensor * src1 = dst->src[1];
|
| 73 |
+
|
| 74 |
+
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
| 75 |
+
GGML_ASSERT(src1->type == GGML_TYPE_F32);
|
| 76 |
+
GGML_ASSERT(dst->type == GGML_TYPE_F32);
|
| 77 |
+
|
| 78 |
+
const int d_conv = src1->ne[0];
|
| 79 |
+
const int ncs = src0->ne[0];
|
| 80 |
+
const int d_inner = src0->ne[1];
|
| 81 |
+
const int n_t = dst->ne[1];
|
| 82 |
+
const int n_s = dst->ne[2];
|
| 83 |
+
|
| 84 |
+
GGML_ASSERT(src0->ne[0] == d_conv - 1 + n_t);
|
| 85 |
+
GGML_ASSERT(src0->ne[1] == d_inner);
|
| 86 |
+
GGML_ASSERT(src1->ne[1] == d_inner);
|
| 87 |
+
|
| 88 |
+
GGML_ASSERT(dst->ne[0] == d_inner);
|
| 89 |
+
GGML_ASSERT(dst->ne[1] == n_t);
|
| 90 |
+
GGML_ASSERT(dst->ne[2] == n_s);
|
| 91 |
+
|
| 92 |
+
GGML_ASSERT(src0->nb[0] == sizeof(float));
|
| 93 |
+
GGML_ASSERT(src1->nb[0] == sizeof(float));
|
| 94 |
+
|
| 95 |
+
GGML_ASSERT(src0->nb[1] == src0->ne[0] * sizeof(float));
|
| 96 |
+
|
| 97 |
+
const int src_stride_inner = ncs;
|
| 98 |
+
const int src_stride_seq = ncs * d_inner;
|
| 99 |
+
const int dst_stride_token = d_inner;
|
| 100 |
+
const int dst_stride_seq = d_inner * n_t;
|
| 101 |
+
|
| 102 |
+
try {
|
| 103 |
+
queue *q = ctx.stream();
|
| 104 |
+
|
| 105 |
+
const float *src_data = static_cast<const float *>(src0->data);
|
| 106 |
+
const float *weights = static_cast<const float *>(src1->data);
|
| 107 |
+
float *dst_data = static_cast<float *>(dst->data);
|
| 108 |
+
|
| 109 |
+
GGML_ASSERT(src_data && weights && dst_data);
|
| 110 |
+
|
| 111 |
+
kernel_ssm_conv(
|
| 112 |
+
*q,
|
| 113 |
+
src_data,
|
| 114 |
+
weights,
|
| 115 |
+
dst_data,
|
| 116 |
+
d_conv,
|
| 117 |
+
d_inner,
|
| 118 |
+
n_t,
|
| 119 |
+
n_s,
|
| 120 |
+
ncs,
|
| 121 |
+
src_stride_inner,
|
| 122 |
+
src_stride_seq,
|
| 123 |
+
dst_stride_token,
|
| 124 |
+
dst_stride_seq
|
| 125 |
+
);
|
| 126 |
+
|
| 127 |
+
} catch (const std::exception &e) {
|
| 128 |
+
std::fprintf(stderr, "[SYCL-SSM_CONV] ERROR: %s\n", e.what());
|
| 129 |
+
throw;
|
| 130 |
+
}
|
| 131 |
+
}
|
| 132 |
+
|
| 133 |
+
void ggml_sycl_ssm_conv(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
|
| 134 |
+
scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/2);
|
| 135 |
+
ggml_sycl_op_ssm_conv(ctx, dst);
|
| 136 |
+
}
|
ggml/src/ggml-sycl/ssm_conv.hpp
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#pragma once
|
| 2 |
+
|
| 3 |
+
#include "common.hpp"
|
| 4 |
+
|
| 5 |
+
void ggml_sycl_ssm_conv(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
|
ggml/src/ggml-sycl/ssm_scan.cpp
ADDED
|
@@ -0,0 +1,170 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#include "ssm_scan.hpp"
|
| 2 |
+
#include "common.hpp"
|
| 3 |
+
|
| 4 |
+
template <int c_factor, int d_state>
|
| 5 |
+
static void ssm_scan_f32_group(
|
| 6 |
+
const float * __restrict__ src0, const float * __restrict__ src1, const float * __restrict__ src2,
|
| 7 |
+
const float * __restrict__ src3, const float * __restrict__ src4, const float * __restrict__ src5,
|
| 8 |
+
const int32_t * __restrict__ src6, float * __restrict__ dst,
|
| 9 |
+
const int src0_nb2, const int src0_nb3, const int src1_nb2, const int src1_nb3,
|
| 10 |
+
const int src2_nb1, const int src2_nb2, const int src3_nb1,
|
| 11 |
+
const int src4_nb2, const int src4_nb3, const int src5_nb2, const int src5_nb3,
|
| 12 |
+
const int64_t s_off, const int64_t n_head, const int64_t d_head, const int64_t n_group, const int64_t n_tok,
|
| 13 |
+
const int64_t K,
|
| 14 |
+
const sycl::nd_item<2> & item) {
|
| 15 |
+
|
| 16 |
+
const int lane = item.get_local_id(1) % WARP_SIZE;
|
| 17 |
+
const int warp = item.get_local_id(1) / WARP_SIZE;
|
| 18 |
+
const int warp_idx = item.get_group(1) * c_factor + warp;
|
| 19 |
+
const int seq_idx = item.get_group(0);
|
| 20 |
+
|
| 21 |
+
const int head_idx = warp_idx / d_head;
|
| 22 |
+
const int head_off = (warp_idx % d_head) * sizeof(float);
|
| 23 |
+
const int group_off = (head_idx / (n_head / n_group)) * d_state * sizeof(float);
|
| 24 |
+
|
| 25 |
+
const float * s0_warp = (const float *) ((const char *) src0 + src6[seq_idx] * src0_nb3 + head_idx * src0_nb2 + head_off * d_state);
|
| 26 |
+
const float * x_warp = (const float *) ((const char *) src1 + (seq_idx * src1_nb3) + (warp_idx * sizeof(float)));
|
| 27 |
+
const float * dt_warp = (const float *) ((const char *) src2 + (seq_idx * src2_nb2) + head_idx * sizeof(float));
|
| 28 |
+
const float * A_warp = (const float *) ((const char *) src3 + head_idx * src3_nb1);
|
| 29 |
+
const float * B_warp = (const float *) ((const char *) src4 + (seq_idx * src4_nb3) + (group_off));
|
| 30 |
+
const float * C_warp = (const float *) ((const char *) src5 + (seq_idx * src5_nb3) + (group_off));
|
| 31 |
+
float * y_warp = dst + (seq_idx * n_tok * n_head * d_head) + warp_idx;
|
| 32 |
+
float * s_warp = (float *) ((char *) dst + s_off + seq_idx * src0_nb3 + head_idx * src0_nb2 + head_off * d_state);
|
| 33 |
+
|
| 34 |
+
const int stride_x = src1_nb2 / sizeof(float);
|
| 35 |
+
const int stride_dt = src2_nb1 / sizeof(float);
|
| 36 |
+
const int stride_B = src4_nb2 / sizeof(float);
|
| 37 |
+
const int stride_C = src5_nb2 / sizeof(float);
|
| 38 |
+
const int stride_y = n_head * d_head;
|
| 39 |
+
|
| 40 |
+
float state[c_factor];
|
| 41 |
+
float state_sum = 0.0f;
|
| 42 |
+
|
| 43 |
+
#pragma unroll
|
| 44 |
+
for (int j = 0; j < c_factor; j++) {
|
| 45 |
+
state[j] = s0_warp[WARP_SIZE * j + lane];
|
| 46 |
+
}
|
| 47 |
+
|
| 48 |
+
for (int64_t i = 0; i < n_tok; i++) {
|
| 49 |
+
const float dt_val = dt_warp[i * stride_dt];
|
| 50 |
+
const float dt_soft_plus = (dt_val <= 20.0f ? sycl::log1p(sycl::exp(dt_val)) : dt_val);
|
| 51 |
+
|
| 52 |
+
state_sum = 0.0f;
|
| 53 |
+
const float dA = sycl::exp(dt_soft_plus * A_warp[0]);
|
| 54 |
+
const float x_dt = x_warp[i * stride_x] * dt_soft_plus;
|
| 55 |
+
#pragma unroll
|
| 56 |
+
for (int j = 0; j < c_factor; j++) {
|
| 57 |
+
const float B_val = B_warp[i * stride_B + WARP_SIZE * j + lane];
|
| 58 |
+
const float C_val = C_warp[i * stride_C + WARP_SIZE * j + lane];
|
| 59 |
+
state[j] = (state[j] * dA) + (B_val * x_dt);
|
| 60 |
+
state_sum += state[j] * C_val;
|
| 61 |
+
}
|
| 62 |
+
|
| 63 |
+
state_sum = warp_reduce_sum<WARP_SIZE>(state_sum);
|
| 64 |
+
|
| 65 |
+
if (lane == 0) {
|
| 66 |
+
y_warp[i * stride_y] = state_sum;
|
| 67 |
+
}
|
| 68 |
+
|
| 69 |
+
const int64_t slot = n_tok - 1 - i;
|
| 70 |
+
if (K > 1 && slot > 0 && slot < K) {
|
| 71 |
+
float * s_snapshot_warp = (float *) ((char *) dst + s_off + (slot * item.get_group_range(0) + seq_idx) * src0_nb3 + head_idx * src0_nb2 + head_off * d_state);
|
| 72 |
+
#pragma unroll
|
| 73 |
+
for (int j = 0; j < c_factor; j++) {
|
| 74 |
+
s_snapshot_warp[WARP_SIZE * j + lane] = state[j];
|
| 75 |
+
}
|
| 76 |
+
}
|
| 77 |
+
}
|
| 78 |
+
|
| 79 |
+
#pragma unroll
|
| 80 |
+
for (int j = 0; j < c_factor; j++) {
|
| 81 |
+
s_warp[WARP_SIZE * j + lane] = state[j];
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
|
| 85 |
+
static void ssm_scan_f32_sycl(
|
| 86 |
+
const float * src0, const float * src1, const float * src2, const float * src3,
|
| 87 |
+
const float * src4, const float * src5, const int32_t * src6, float * dst,
|
| 88 |
+
const int src0_nb2, const int src0_nb3, const int src1_nb2, const int src1_nb3, const int src2_nb1,
|
| 89 |
+
const int src2_nb2, const int src3_nb1, const int src4_nb2, const int src4_nb3, const int src5_nb2,
|
| 90 |
+
const int src5_nb3, const int64_t s_off, const int64_t d_state, const int64_t head_dim,
|
| 91 |
+
const int64_t n_head, const int64_t n_group, const int64_t n_tok, const int64_t n_seq,
|
| 92 |
+
const int64_t K,
|
| 93 |
+
dpct::queue_ptr stream) {
|
| 94 |
+
|
| 95 |
+
// NOTE: if you change conditions here, be sure to update the corresponding supports_op condition!
|
| 96 |
+
GGML_ASSERT(src3_nb1 == sizeof(float));
|
| 97 |
+
if (d_state == 128) {
|
| 98 |
+
constexpr int threads = 128;
|
| 99 |
+
constexpr int num_warps = threads / WARP_SIZE;
|
| 100 |
+
const sycl::range<2> grid(n_seq, (n_head * head_dim + num_warps - 1) / num_warps);
|
| 101 |
+
const sycl::range<2> block(1, threads);
|
| 102 |
+
stream->parallel_for(
|
| 103 |
+
sycl::nd_range<2>(grid * block, block),
|
| 104 |
+
[=](sycl::nd_item<2> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
| 105 |
+
ssm_scan_f32_group<128 / WARP_SIZE, 128>(
|
| 106 |
+
src0, src1, src2, src3, src4, src5, src6, dst,
|
| 107 |
+
src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2, src3_nb1,
|
| 108 |
+
src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, head_dim, n_group, n_tok, K, item);
|
| 109 |
+
});
|
| 110 |
+
} else if (d_state == 256) {
|
| 111 |
+
constexpr int threads = 256;
|
| 112 |
+
constexpr int num_warps = threads / WARP_SIZE;
|
| 113 |
+
const sycl::range<2> grid(n_seq, (n_head * head_dim + num_warps - 1) / num_warps);
|
| 114 |
+
const sycl::range<2> block(1, threads);
|
| 115 |
+
stream->parallel_for(
|
| 116 |
+
sycl::nd_range<2>(grid * block, block),
|
| 117 |
+
[=](sycl::nd_item<2> item) [[sycl::reqd_sub_group_size(WARP_SIZE)]] {
|
| 118 |
+
ssm_scan_f32_group<256 / WARP_SIZE, 256>(
|
| 119 |
+
src0, src1, src2, src3, src4, src5, src6, dst,
|
| 120 |
+
src0_nb2, src0_nb3, src1_nb2, src1_nb3, src2_nb1, src2_nb2, src3_nb1,
|
| 121 |
+
src4_nb2, src4_nb3, src5_nb2, src5_nb3, s_off, n_head, head_dim, n_group, n_tok, K, item);
|
| 122 |
+
});
|
| 123 |
+
} else {
|
| 124 |
+
GGML_ABORT("ssm_scan: unsupported d_state (must be 128 or 256)");
|
| 125 |
+
}
|
| 126 |
+
}
|
| 127 |
+
|
| 128 |
+
inline void ggml_sycl_op_ssm_scan(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
|
| 129 |
+
const ggml_tensor * src0 = dst->src[0];
|
| 130 |
+
const ggml_tensor * src1 = dst->src[1];
|
| 131 |
+
const ggml_tensor * src2 = dst->src[2];
|
| 132 |
+
const ggml_tensor * src3 = dst->src[3];
|
| 133 |
+
const ggml_tensor * src4 = dst->src[4];
|
| 134 |
+
const ggml_tensor * src5 = dst->src[5];
|
| 135 |
+
const ggml_tensor * src6 = dst->src[6];
|
| 136 |
+
|
| 137 |
+
GGML_ASSERT(src0->type == GGML_TYPE_F32);
|
| 138 |
+
GGML_ASSERT(src6->type == GGML_TYPE_I32);
|
| 139 |
+
GGML_ASSERT(dst->type == GGML_TYPE_F32);
|
| 140 |
+
|
| 141 |
+
const int64_t nc = src0->ne[0];
|
| 142 |
+
const int64_t nr = src0->ne[1];
|
| 143 |
+
const int64_t nh = src1->ne[1];
|
| 144 |
+
const int64_t ng = src4->ne[1];
|
| 145 |
+
const int64_t n_t = src1->ne[2];
|
| 146 |
+
const int64_t n_s = src1->ne[3];
|
| 147 |
+
const int64_t K = ggml_get_op_params_i32(dst, 0);
|
| 148 |
+
const int64_t s_off = ggml_nelements(src1) * sizeof(float);
|
| 149 |
+
|
| 150 |
+
GGML_ASSERT(K >= 1);
|
| 151 |
+
GGML_ASSERT(ggml_nelements(src1) + K * nc * nr * nh * n_s == ggml_nelements(dst));
|
| 152 |
+
GGML_ASSERT(src3->ne[0] == 1 || K == 1);
|
| 153 |
+
|
| 154 |
+
dpct::queue_ptr stream = ctx.stream();
|
| 155 |
+
SYCL_CHECK(ggml_sycl_set_device(ctx.device));
|
| 156 |
+
|
| 157 |
+
ssm_scan_f32_sycl(
|
| 158 |
+
static_cast<const float *>(src0->data), static_cast<const float *>(src1->data),
|
| 159 |
+
static_cast<const float *>(src2->data), static_cast<const float *>(src3->data),
|
| 160 |
+
static_cast<const float *>(src4->data), static_cast<const float *>(src5->data),
|
| 161 |
+
static_cast<const int32_t *>(src6->data), static_cast<float *>(dst->data),
|
| 162 |
+
src0->nb[2], src0->nb[3], src1->nb[2], src1->nb[3], src2->nb[1], src2->nb[2],
|
| 163 |
+
src3->nb[1], src4->nb[2], src4->nb[3], src5->nb[2], src5->nb[3],
|
| 164 |
+
s_off, nc, nr, nh, ng, n_t, n_s, K, stream);
|
| 165 |
+
}
|
| 166 |
+
|
| 167 |
+
void ggml_sycl_ssm_scan(ggml_backend_sycl_context & ctx, ggml_tensor * dst) {
|
| 168 |
+
scope_op_debug_print scope_dbg_print(__func__, dst, /*num_src=*/7);
|
| 169 |
+
ggml_sycl_op_ssm_scan(ctx, dst);
|
| 170 |
+
}
|
ggml/src/ggml-sycl/ssm_scan.hpp
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#pragma once
|
| 2 |
+
|
| 3 |
+
#include "common.hpp"
|
| 4 |
+
|
| 5 |
+
void ggml_sycl_ssm_scan(ggml_backend_sycl_context & ctx, ggml_tensor * dst);
|
ggml/src/ggml-sycl/sycl_hw.cpp
ADDED
|
@@ -0,0 +1,67 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#include "sycl_hw.hpp"
|
| 2 |
+
|
| 3 |
+
using namespace std;
|
| 4 |
+
|
| 5 |
+
/*defined in
|
| 6 |
+
* /opt/intel/oneapi/compiler/latest/include/sycl/ext/oneapi/experimental/device_architecture.def
|
| 7 |
+
*/
|
| 8 |
+
static map<gpu_arch, std::pair<const char*, sycl_intel_gpu_family>> arch2name = {
|
| 9 |
+
{gpu_arch::intel_gpu_bdw, {"intel_gpu_bdw", GPU_FAMILY_IGPU_NON_XE}},
|
| 10 |
+
{gpu_arch::intel_gpu_skl, {"intel_gpu_skl", GPU_FAMILY_IGPU_NON_XE}},
|
| 11 |
+
{gpu_arch::intel_gpu_kbl, {"intel_gpu_kbl", GPU_FAMILY_IGPU_NON_XE}},
|
| 12 |
+
{gpu_arch::intel_gpu_cfl, {"intel_gpu_cfl", GPU_FAMILY_IGPU_NON_XE}},
|
| 13 |
+
{gpu_arch::intel_gpu_apl, {"intel_gpu_apl", GPU_FAMILY_IGPU_NON_XE}},
|
| 14 |
+
{gpu_arch::intel_gpu_glk, {"intel_gpu_glk", GPU_FAMILY_IGPU_NON_XE}},
|
| 15 |
+
{gpu_arch::intel_gpu_whl, {"intel_gpu_whl", GPU_FAMILY_IGPU_NON_XE}},
|
| 16 |
+
{gpu_arch::intel_gpu_aml, {"intel_gpu_aml", GPU_FAMILY_IGPU_NON_XE}},
|
| 17 |
+
{gpu_arch::intel_gpu_cml, {"intel_gpu_cml", GPU_FAMILY_IGPU_NON_XE}},
|
| 18 |
+
{gpu_arch::intel_gpu_icllp, {"intel_gpu_icllp", GPU_FAMILY_IGPU_NON_XE}},
|
| 19 |
+
{gpu_arch::intel_gpu_ehl, {"intel_gpu_ehl", GPU_FAMILY_IGPU_NON_XE}},
|
| 20 |
+
{gpu_arch::intel_gpu_tgllp, {"intel_gpu_tgllp", GPU_FAMILY_IGPU_NON_XE}},
|
| 21 |
+
{gpu_arch::intel_gpu_rkl, {"intel_gpu_rkl", GPU_FAMILY_IGPU_NON_XE}},
|
| 22 |
+
{gpu_arch::intel_gpu_adl_s, {"intel_gpu_adl_s", GPU_FAMILY_IGPU_NON_XE}},
|
| 23 |
+
{gpu_arch::intel_gpu_adl_p, {"intel_gpu_adl_p", GPU_FAMILY_IGPU_NON_XE}},
|
| 24 |
+
{gpu_arch::intel_gpu_adl_n, {"intel_gpu_adl_n", GPU_FAMILY_IGPU_NON_XE}},
|
| 25 |
+
{gpu_arch::intel_gpu_dg1, {"intel_gpu_dg1", GPU_FAMILY_DGPU_CLIENT_GAME}},
|
| 26 |
+
{gpu_arch::intel_gpu_acm_g10, {"intel_gpu_acm_g10", GPU_FAMILY_DGPU_CLIENT_GAME}},
|
| 27 |
+
{gpu_arch::intel_gpu_acm_g11, {"intel_gpu_acm_g11", GPU_FAMILY_DGPU_CLIENT_GAME}},
|
| 28 |
+
{gpu_arch::intel_gpu_acm_g12, {"intel_gpu_acm_g12", GPU_FAMILY_DGPU_CLIENT_GAME}},
|
| 29 |
+
{gpu_arch::intel_gpu_pvc, {"intel_gpu_pvc", GPU_FAMILY_DGPU_CLOUD}},
|
| 30 |
+
{gpu_arch::intel_gpu_pvc_vg, {"intel_gpu_pvc_vg", GPU_FAMILY_DGPU_CLOUD}},
|
| 31 |
+
{gpu_arch::intel_gpu_mtl_u, {"intel_gpu_mtl_u", GPU_FAMILY_IGPU_XE}},
|
| 32 |
+
{gpu_arch::intel_gpu_mtl_h, {"intel_gpu_mtl_h", GPU_FAMILY_IGPU_XE}},
|
| 33 |
+
{gpu_arch::intel_gpu_arl_h, {"intel_gpu_arl_h", GPU_FAMILY_IGPU_XE}},
|
| 34 |
+
{gpu_arch::intel_gpu_bmg_g21, {"intel_gpu_bmg_g21", GPU_FAMILY_DGPU_CLIENT_GAME}},
|
| 35 |
+
{gpu_arch::intel_gpu_bmg_g31, {"intel_gpu_bmg_g31", GPU_FAMILY_DGPU_CLIENT_GAME}},
|
| 36 |
+
{gpu_arch::intel_gpu_lnl_m, {"intel_gpu_lnl_m", GPU_FAMILY_IGPU_XE}},
|
| 37 |
+
{gpu_arch::intel_gpu_ptl_h, {"intel_gpu_ptl_h", GPU_FAMILY_IGPU_XE}},
|
| 38 |
+
{gpu_arch::intel_gpu_ptl_u, {"intel_gpu_ptl_u", GPU_FAMILY_IGPU_XE}},
|
| 39 |
+
{gpu_arch::intel_gpu_wcl, {"intel_gpu_wcl", GPU_FAMILY_IGPU_XE}}
|
| 40 |
+
};
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
sycl_hw_info get_device_hw_info(sycl::device* device_ptr) {
|
| 44 |
+
sycl_hw_info res;
|
| 45 |
+
int32_t id =
|
| 46 |
+
device_ptr->get_info<sycl::ext::intel::info::device::device_id>();
|
| 47 |
+
res.device_id = id;
|
| 48 |
+
|
| 49 |
+
res.name = device_ptr->get_info<sycl::info::device::name>();
|
| 50 |
+
|
| 51 |
+
syclex::architecture arch =
|
| 52 |
+
device_ptr->get_info<syclex::info::device::architecture>();
|
| 53 |
+
res.arch = arch;
|
| 54 |
+
|
| 55 |
+
map<syclex::architecture,
|
| 56 |
+
std::pair<const char*, sycl_intel_gpu_family>>::iterator it =
|
| 57 |
+
arch2name.find(res.arch);
|
| 58 |
+
if (it != arch2name.end()) {
|
| 59 |
+
res.arch_name = it->second.first;
|
| 60 |
+
res.gpu_family = it->second.second;
|
| 61 |
+
} else {
|
| 62 |
+
res.arch_name = "unknown";
|
| 63 |
+
res.gpu_family = GPU_FAMILY_UKNOWN;
|
| 64 |
+
}
|
| 65 |
+
|
| 66 |
+
return res;
|
| 67 |
+
}
|
ggml/src/ggml-sycl/sycl_hw.hpp
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#ifndef SYCL_HW_HPP
|
| 2 |
+
#define SYCL_HW_HPP
|
| 3 |
+
|
| 4 |
+
#include <algorithm>
|
| 5 |
+
#include <stdio.h>
|
| 6 |
+
#include <vector>
|
| 7 |
+
#include <map>
|
| 8 |
+
|
| 9 |
+
#include <sycl/sycl.hpp>
|
| 10 |
+
|
| 11 |
+
namespace syclex = sycl::ext::oneapi::experimental;
|
| 12 |
+
using gpu_arch = sycl::ext::oneapi::experimental::architecture;
|
| 13 |
+
|
| 14 |
+
// It's used to mark the GPU computing capacity
|
| 15 |
+
// The value must flow the order of performance.
|
| 16 |
+
enum sycl_intel_gpu_family {
|
| 17 |
+
GPU_FAMILY_UKNOWN = -1,
|
| 18 |
+
// iGPU without Xe core, before Meteor Lake iGPU(Xe)
|
| 19 |
+
GPU_FAMILY_IGPU_NON_XE = 0,
|
| 20 |
+
// iGPU with Xe core, Meteor Lake iGPU or newer.
|
| 21 |
+
GPU_FAMILY_IGPU_XE = 1,
|
| 22 |
+
// dGPU for gaming in client/data center (DG1/FLex 140 or newer).
|
| 23 |
+
GPU_FAMILY_DGPU_CLIENT_GAME = 2,
|
| 24 |
+
// dGPU for AI in cloud, PVC or newer.
|
| 25 |
+
GPU_FAMILY_DGPU_CLOUD = 3
|
| 26 |
+
};
|
| 27 |
+
|
| 28 |
+
struct sycl_hw_info {
|
| 29 |
+
syclex::architecture arch;
|
| 30 |
+
const char* arch_name;
|
| 31 |
+
int32_t device_id;
|
| 32 |
+
std::string name;
|
| 33 |
+
sycl_intel_gpu_family gpu_family;
|
| 34 |
+
};
|
| 35 |
+
|
| 36 |
+
sycl_hw_info get_device_hw_info(sycl::device *device_ptr);
|
| 37 |
+
|
| 38 |
+
#endif // SYCL_HW_HPP
|
ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq112-dv112.cpp
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// This file has been autogenerated by generate_cu_files.py, do not edit manually.
|
| 2 |
+
|
| 3 |
+
#include "../fattn-tile.hpp"
|
| 4 |
+
|
| 5 |
+
DECL_FATTN_TILE_CASE(112, 112);
|
ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq128-dv128.cpp
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// This file has been autogenerated by generate_cu_files.py, do not edit manually.
|
| 2 |
+
|
| 3 |
+
#include "../fattn-tile.hpp"
|
| 4 |
+
|
| 5 |
+
DECL_FATTN_TILE_CASE(128, 128);
|
ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq256-dv256.cpp
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// This file has been autogenerated by generate_cu_files.py, do not edit manually.
|
| 2 |
+
|
| 3 |
+
#include "../fattn-tile.hpp"
|
| 4 |
+
|
| 5 |
+
DECL_FATTN_TILE_CASE(256, 256);
|
ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq40-dv40.cpp
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// This file has been autogenerated by generate_cu_files.py, do not edit manually.
|
| 2 |
+
|
| 3 |
+
#include "../fattn-tile.hpp"
|
| 4 |
+
|
| 5 |
+
DECL_FATTN_TILE_CASE(40, 40);
|
ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq512-dv512.cpp
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// This file has been autogenerated by generate_cu_files.py, do not edit manually.
|
| 2 |
+
|
| 3 |
+
#include "../fattn-tile.hpp"
|
| 4 |
+
|
| 5 |
+
DECL_FATTN_TILE_CASE(512, 512);
|
| 6 |
+
|
ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq576-dv512.cpp
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// This file has been autogenerated by generate_cu_files.py, do not edit manually.
|
| 2 |
+
|
| 3 |
+
#include "../fattn-tile.hpp"
|
| 4 |
+
|
| 5 |
+
DECL_FATTN_TILE_CASE(576, 512);
|
ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq64-dv64.cpp
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// This file has been autogenerated by generate_cu_files.py, do not edit manually.
|
| 2 |
+
|
| 3 |
+
#include "../fattn-tile.hpp"
|
| 4 |
+
|
| 5 |
+
DECL_FATTN_TILE_CASE(64, 64);
|
ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq72-dv72.cpp
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// This file has been autogenerated by generate_cu_files.py, do not edit manually.
|
| 2 |
+
|
| 3 |
+
#include "../fattn-tile.hpp"
|
| 4 |
+
|
| 5 |
+
DECL_FATTN_TILE_CASE(72, 72);
|
ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq80-dv80.cpp
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// This file has been autogenerated by generate_cu_files.py, do not edit manually.
|
| 2 |
+
|
| 3 |
+
#include "../fattn-tile.hpp"
|
| 4 |
+
|
| 5 |
+
DECL_FATTN_TILE_CASE(80, 80);
|
ggml/src/ggml-sycl/template-instances/fattn-tile-instance-dkq96-dv96.cpp
ADDED
|
@@ -0,0 +1,5 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// This file has been autogenerated by generate_cu_files.py, do not edit manually.
|
| 2 |
+
|
| 3 |
+
#include "../fattn-tile.hpp"
|
| 4 |
+
|
| 5 |
+
DECL_FATTN_TILE_CASE(96, 96);
|
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-f16.cpp
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// This file has been autogenerated by generate_cu_files.py, do not edit manually.
|
| 2 |
+
|
| 3 |
+
#include "../fattn-vec.hpp"
|
| 4 |
+
|
| 5 |
+
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F16, GGML_TYPE_F16);
|
| 6 |
+
DECL_FATTN_VEC_CASE(128, GGML_TYPE_F16, GGML_TYPE_F16);
|
| 7 |
+
DECL_FATTN_VEC_CASE(256, GGML_TYPE_F16, GGML_TYPE_F16);
|
| 8 |
+
DECL_FATTN_VEC_CASE(512, GGML_TYPE_F16, GGML_TYPE_F16);
|
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_0.cpp
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// This file has been autogenerated by generate_cu_files.py, do not edit manually.
|
| 2 |
+
|
| 3 |
+
#include "../fattn-vec.hpp"
|
| 4 |
+
|
| 5 |
+
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q4_0);
|
| 6 |
+
DECL_FATTN_VEC_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q4_0);
|
| 7 |
+
DECL_FATTN_VEC_CASE(256, GGML_TYPE_F16, GGML_TYPE_Q4_0);
|
| 8 |
+
DECL_FATTN_VEC_CASE(512, GGML_TYPE_F16, GGML_TYPE_Q4_0);
|
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q4_1.cpp
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// This file has been autogenerated by generate_cu_files.py, do not edit manually.
|
| 2 |
+
|
| 3 |
+
#include "../fattn-vec.hpp"
|
| 4 |
+
|
| 5 |
+
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q4_1);
|
| 6 |
+
DECL_FATTN_VEC_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q4_1);
|
| 7 |
+
DECL_FATTN_VEC_CASE(256, GGML_TYPE_F16, GGML_TYPE_Q4_1);
|
| 8 |
+
DECL_FATTN_VEC_CASE(512, GGML_TYPE_F16, GGML_TYPE_Q4_1);
|
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_0.cpp
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// This file has been autogenerated by generate_cu_files.py, do not edit manually.
|
| 2 |
+
|
| 3 |
+
#include "../fattn-vec.hpp"
|
| 4 |
+
|
| 5 |
+
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q5_0);
|
| 6 |
+
DECL_FATTN_VEC_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q5_0);
|
| 7 |
+
DECL_FATTN_VEC_CASE(256, GGML_TYPE_F16, GGML_TYPE_Q5_0);
|
| 8 |
+
DECL_FATTN_VEC_CASE(512, GGML_TYPE_F16, GGML_TYPE_Q5_0);
|
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q5_1.cpp
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// This file has been autogenerated by generate_cu_files.py, do not edit manually.
|
| 2 |
+
|
| 3 |
+
#include "../fattn-vec.hpp"
|
| 4 |
+
|
| 5 |
+
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q5_1);
|
| 6 |
+
DECL_FATTN_VEC_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q5_1);
|
| 7 |
+
DECL_FATTN_VEC_CASE(256, GGML_TYPE_F16, GGML_TYPE_Q5_1);
|
| 8 |
+
DECL_FATTN_VEC_CASE(512, GGML_TYPE_F16, GGML_TYPE_Q5_1);
|
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-f16-q8_0.cpp
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// This file has been autogenerated by generate_cu_files.py, do not edit manually.
|
| 2 |
+
|
| 3 |
+
#include "../fattn-vec.hpp"
|
| 4 |
+
|
| 5 |
+
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_F16, GGML_TYPE_Q8_0);
|
| 6 |
+
DECL_FATTN_VEC_CASE(128, GGML_TYPE_F16, GGML_TYPE_Q8_0);
|
| 7 |
+
DECL_FATTN_VEC_CASE(256, GGML_TYPE_F16, GGML_TYPE_Q8_0);
|
| 8 |
+
DECL_FATTN_VEC_CASE(512, GGML_TYPE_F16, GGML_TYPE_Q8_0);
|
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-f16.cpp
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// This file has been autogenerated by generate_cu_files.py, do not edit manually.
|
| 2 |
+
|
| 3 |
+
#include "../fattn-vec.hpp"
|
| 4 |
+
|
| 5 |
+
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_0, GGML_TYPE_F16);
|
| 6 |
+
DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_F16);
|
| 7 |
+
DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_0, GGML_TYPE_F16);
|
| 8 |
+
DECL_FATTN_VEC_CASE(512, GGML_TYPE_Q4_0, GGML_TYPE_F16);
|
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_0.cpp
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// This file has been autogenerated by generate_cu_files.py, do not edit manually.
|
| 2 |
+
|
| 3 |
+
#include "../fattn-vec.hpp"
|
| 4 |
+
|
| 5 |
+
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0);
|
| 6 |
+
DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0);
|
| 7 |
+
DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0);
|
| 8 |
+
DECL_FATTN_VEC_CASE(512, GGML_TYPE_Q4_0, GGML_TYPE_Q4_0);
|
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q4_1.cpp
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// This file has been autogenerated by generate_cu_files.py, do not edit manually.
|
| 2 |
+
|
| 3 |
+
#include "../fattn-vec.hpp"
|
| 4 |
+
|
| 5 |
+
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1);
|
| 6 |
+
DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1);
|
| 7 |
+
DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1);
|
| 8 |
+
DECL_FATTN_VEC_CASE(512, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1);
|
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_0.cpp
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// This file has been autogenerated by generate_cu_files.py, do not edit manually.
|
| 2 |
+
|
| 3 |
+
#include "../fattn-vec.hpp"
|
| 4 |
+
|
| 5 |
+
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_0, GGML_TYPE_Q5_0);
|
| 6 |
+
DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q5_0);
|
| 7 |
+
DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_0, GGML_TYPE_Q5_0);
|
| 8 |
+
DECL_FATTN_VEC_CASE(512, GGML_TYPE_Q4_0, GGML_TYPE_Q5_0);
|
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q5_1.cpp
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// This file has been autogenerated by generate_cu_files.py, do not edit manually.
|
| 2 |
+
|
| 3 |
+
#include "../fattn-vec.hpp"
|
| 4 |
+
|
| 5 |
+
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_0, GGML_TYPE_Q5_1);
|
| 6 |
+
DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q5_1);
|
| 7 |
+
DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_0, GGML_TYPE_Q5_1);
|
| 8 |
+
DECL_FATTN_VEC_CASE(512, GGML_TYPE_Q4_0, GGML_TYPE_Q5_1);
|
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_0-q8_0.cpp
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// This file has been autogenerated by generate_cu_files.py, do not edit manually.
|
| 2 |
+
|
| 3 |
+
#include "../fattn-vec.hpp"
|
| 4 |
+
|
| 5 |
+
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0);
|
| 6 |
+
DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0);
|
| 7 |
+
DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0);
|
| 8 |
+
DECL_FATTN_VEC_CASE(512, GGML_TYPE_Q4_0, GGML_TYPE_Q8_0);
|
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-f16.cpp
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// This file has been autogenerated by generate_cu_files.py, do not edit manually.
|
| 2 |
+
|
| 3 |
+
#include "../fattn-vec.hpp"
|
| 4 |
+
|
| 5 |
+
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_1, GGML_TYPE_F16);
|
| 6 |
+
DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_F16);
|
| 7 |
+
DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_1, GGML_TYPE_F16);
|
| 8 |
+
DECL_FATTN_VEC_CASE(512, GGML_TYPE_Q4_1, GGML_TYPE_F16);
|
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_0.cpp
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// This file has been autogenerated by generate_cu_files.py, do not edit manually.
|
| 2 |
+
|
| 3 |
+
#include "../fattn-vec.hpp"
|
| 4 |
+
|
| 5 |
+
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0);
|
| 6 |
+
DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0);
|
| 7 |
+
DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0);
|
| 8 |
+
DECL_FATTN_VEC_CASE(512, GGML_TYPE_Q4_1, GGML_TYPE_Q4_0);
|
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q4_1.cpp
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// This file has been autogenerated by generate_cu_files.py, do not edit manually.
|
| 2 |
+
|
| 3 |
+
#include "../fattn-vec.hpp"
|
| 4 |
+
|
| 5 |
+
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1);
|
| 6 |
+
DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1);
|
| 7 |
+
DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1);
|
| 8 |
+
DECL_FATTN_VEC_CASE(512, GGML_TYPE_Q4_1, GGML_TYPE_Q4_1);
|
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_0.cpp
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// This file has been autogenerated by generate_cu_files.py, do not edit manually.
|
| 2 |
+
|
| 3 |
+
#include "../fattn-vec.hpp"
|
| 4 |
+
|
| 5 |
+
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0);
|
| 6 |
+
DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0);
|
| 7 |
+
DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0);
|
| 8 |
+
DECL_FATTN_VEC_CASE(512, GGML_TYPE_Q4_1, GGML_TYPE_Q5_0);
|
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q5_1.cpp
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// This file has been autogenerated by generate_cu_files.py, do not edit manually.
|
| 2 |
+
|
| 3 |
+
#include "../fattn-vec.hpp"
|
| 4 |
+
|
| 5 |
+
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_1, GGML_TYPE_Q5_1);
|
| 6 |
+
DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q5_1);
|
| 7 |
+
DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_1, GGML_TYPE_Q5_1);
|
| 8 |
+
DECL_FATTN_VEC_CASE(512, GGML_TYPE_Q4_1, GGML_TYPE_Q5_1);
|
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q4_1-q8_0.cpp
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// This file has been autogenerated by generate_cu_files.py, do not edit manually.
|
| 2 |
+
|
| 3 |
+
#include "../fattn-vec.hpp"
|
| 4 |
+
|
| 5 |
+
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q4_1, GGML_TYPE_Q8_0);
|
| 6 |
+
DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q4_1, GGML_TYPE_Q8_0);
|
| 7 |
+
DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q4_1, GGML_TYPE_Q8_0);
|
| 8 |
+
DECL_FATTN_VEC_CASE(512, GGML_TYPE_Q4_1, GGML_TYPE_Q8_0);
|
ggml/src/ggml-sycl/template-instances/fattn-vec-instance-q5_0-f16.cpp
ADDED
|
@@ -0,0 +1,8 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// This file has been autogenerated by generate_cu_files.py, do not edit manually.
|
| 2 |
+
|
| 3 |
+
#include "../fattn-vec.hpp"
|
| 4 |
+
|
| 5 |
+
DECL_FATTN_VEC_CASE( 64, GGML_TYPE_Q5_0, GGML_TYPE_F16);
|
| 6 |
+
DECL_FATTN_VEC_CASE(128, GGML_TYPE_Q5_0, GGML_TYPE_F16);
|
| 7 |
+
DECL_FATTN_VEC_CASE(256, GGML_TYPE_Q5_0, GGML_TYPE_F16);
|
| 8 |
+
DECL_FATTN_VEC_CASE(512, GGML_TYPE_Q5_0, GGML_TYPE_F16);
|