lingtiny30vq3 / vq3 /vq3_codebook.h
HelloSun's picture
LUT-VQ3-64: machine-tuned (threads/NUMA/ubatch), full 128k ctx data, one-click build, results
511ba50 verified
Raw History Blame Contribute Delete
9.11 kB
#ifndef GGML_VQ3_CODEBOOK_H
#define GGML_VQ3_CODEBOOK_H
#include <stdint.h>
#define GGML_VQ3_G (8.6373016238e-03f)
#if defined(_MSC_VER)
#define GGML_VQ3_ALIGN64 __declspec(align(64))
#else
#define GGML_VQ3_ALIGN64 __attribute__((aligned(64)))
#endif
static const int8_t ggml_vq3_cb[128][3] = {
{ -24, -19, -24 },
{ -37, -20, -123 },
{ -41, -18, 18 },
{ 23, -6, 27 },
{ 29, -15, -65 },
{ -3, 4, 0 },
{ -59, -25, 72 },
{ -68, -13, -66 },
{ 11, -54, -122 },
{ -84, -5, 30 },
{ 35, -51, 51 },
{ -19, 8, 50 },
{ 54, 9, 62 },
{ 12, -74, -55 },
{ -41, -73, -34 },
{ 16, 46, 19 },
{ -62, -70, -111 },
{ 69, -14, 116 },
{ -65, -73, 20 },
{ 70, 11, 12 },
{ 75, -2, -40 },
{ 55, 43, -65 },
{ -6, 62, -121 },
{ -47, 32, 16 },
{ 73, -64, 65 },
{ -22, 67, 121 },
{ 62, 61, -9 },
{ 5, 80, -18 },
{ -123, -46, 55 },
{ -119, -58, -56 },
{ -24, -120, -122 },
{ -57, 48, -72 },
{ -125, -52, -3 },
{ -30, 77, 27 },
{ -19, -126, -11 },
{ 43, -118, -118 },
{ -56, 69, -22 },
{ 52, -122, 119 },
{ 120, 6, -116 },
{ 123, -22, -59 },
{ 124, -26, 115 },
{ 127, 4, -10 },
{ -97, -121, -34 },
{ 119, 46, -48 },
{ 117, 54, -115 },
{ 7, 117, 76 },
{ 9, 120, 26 },
{ -122, -113, 102 },
{ -123, -114, 19 },
{ 5, 124, -58 },
{ 51, 122, -8 },
{ 100, -123, 51 },
{ 114, -111, -62 },
{ 62, 123, 52 },
{ -56, 119, -59 },
{ -122, 6, -31 },
{ -123, 8, 120 },
{ -121, 40, -121 },
{ 122, -120, -3 },
{ -127, 1, 15 },
{ 114, 114, 111 },
{ 121, 114, 28 },
{ -114, 97, 118 },
{ -119, 116, 48 },
{ 5, -19, 75 },
{ 16, -14, 126 },
{ 10, 8, -122 },
{ -18, -41, -70 },
{ 18, -35, -16 },
{ -13, -51, 18 },
{ -37, -47, 123 },
{ 29, 16, -25 },
{ -15, 8, -63 },
{ -15, 14, 120 },
{ -19, -71, 69 },
{ -75, -42, -18 },
{ 31, -64, 121 },
{ 60, -2, -118 },
{ -15, 37, -26 },
{ 1, 49, 66 },
{ 20, -81, 0 },
{ 43, 42, 120 },
{ 66, -41, 8 },
{ 63, -53, -98 },
{ 4, 58, -67 },
{ 63, -62, -38 },
{ -115, -30, -124 },
{ -108, -47, 118 },
{ 53, 59, -124 },
{ -54, 31, -125 },
{ 58, 65, 56 },
{ -62, 19, 123 },
{ -5, -119, 115 },
{ -66, 13, -23 },
{ -24, -121, 38 },
{ -31, -120, -66 },
{ -61, -113, 118 },
{ -63, 46, 65 },
{ 30, -121, 48 },
{ 35, -124, -54 },
{ 118, -18, 34 },
{ -76, -122, 46 },
{ 56, -122, -4 },
{ 118, 26, 62 },
{ 122, -48, -117 },
{ 115, 44, 120 },
{ 122, -56, -12 },
{ 38, 112, 124 },
{ -113, -115, -109 },
{ 124, -70, 44 },
{ 121, 55, 14 },
{ 63, 110, -59 },
{ 25, 123, -120 },
{ -47, 113, -121 },
{ -24, 124, -9 },
{ 116, -106, 119 },
{ -51, 117, 60 },
{ -40, 124, 122 },
{ 114, -118, -124 },
{ -115, 55, 13 },
{ -124, 4, -76 },
{ -124, 26, 64 },
{ -74, 117, 8 },
{ -119, 59, -46 },
{ 110, 120, -115 },
{ 121, 116, -38 },
{ -115, 114, -106 },
{ -122, 121, -25 },
};
GGML_VQ3_ALIGN64 static const int8_t ggml_vq3_lut0_lo[64] = { -24, -37, -41, 23, 29, -3, -59, -68, 11, -84, 35, -19, 54, 12, -41, 16, -62, 69, -65, 70, 75, 55, -6, -47, 73, -22, 62, 5, -123, -119, -24, -57, -125, -30, -19, 43, -56, 52, 120, 123, 124, 127, -97, 119, 117, 7, 9, -122, -123, 5, 51, 100, 114, 62, -56, -122, -123, -121, 122, -127, 114, 121, -114, -119 };
GGML_VQ3_ALIGN64 static const int8_t ggml_vq3_lut0_hi[64] = { 5, 16, 10, -18, 18, -13, -37, 29, -15, -15, -19, -75, 31, 60, -15, 1, 20, 43, 66, 63, 4, 63, -115, -108, 53, -54, 58, -62, -5, -66, -24, -31, -61, -63, 30, 35, 118, -76, 56, 118, 122, 115, 122, 38, -113, 124, 121, 63, 25, -47, -24, 116, -51, -40, 114, -115, -124, -124, -74, -119, 110, 121, -115, -122 };
GGML_VQ3_ALIGN64 static const int8_t ggml_vq3_lut1_lo[64] = { -19, -20, -18, -6, -15, 4, -25, -13, -54, -5, -51, 8, 9, -74, -73, 46, -70, -14, -73, 11, -2, 43, 62, 32, -64, 67, 61, 80, -46, -58, -120, 48, -52, 77, -126, -118, 69, -122, 6, -22, -26, 4, -121, 46, 54, 117, 120, -113, -114, 124, 122, -123, -111, 123, 119, 6, 8, 40, -120, 1, 114, 114, 97, 116 };
GGML_VQ3_ALIGN64 static const int8_t ggml_vq3_lut1_hi[64] = { -19, -14, 8, -41, -35, -51, -47, 16, 8, 14, -71, -42, -64, -2, 37, 49, -81, 42, -41, -53, 58, -62, -30, -47, 59, 31, 65, 19, -119, 13, -121, -120, -113, 46, -121, -124, -18, -122, -122, 26, -48, 44, -56, 112, -115, -70, 55, 110, 123, 113, 124, -106, 117, 124, -118, 55, 4, 26, 117, 59, 120, 116, 114, 121 };
GGML_VQ3_ALIGN64 static const int8_t ggml_vq3_lut2_lo[64] = { -24, -123, 18, 27, -65, 0, 72, -66, -122, 30, 51, 50, 62, -55, -34, 19, -111, 116, 20, 12, -40, -65, -121, 16, 65, 121, -9, -18, 55, -56, -122, -72, -3, 27, -11, -118, -22, 119, -116, -59, 115, -10, -34, -48, -115, 76, 26, 102, 19, -58, -8, 51, -62, 52, -59, -31, 120, -121, -3, 15, 111, 28, 118, 48 };
GGML_VQ3_ALIGN64 static const int8_t ggml_vq3_lut2_hi[64] = { 75, 126, -122, -70, -16, 18, 123, -25, -63, 120, 69, -18, 121, -118, -26, 66, 0, 120, 8, -98, -67, -38, -124, 118, -124, -125, 56, 123, 115, -23, 38, -66, 118, 65, 48, -54, 34, 46, -4, 62, -117, 120, -12, 124, -109, 44, 14, -59, -120, -121, -9, 119, 60, 122, -124, 13, -76, 64, 8, -46, -115, -38, -106, -25 };
GGML_VQ3_ALIGN64 static const uint8_t ggml_vq3_perm_rep64[64] = {
0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19,20,21,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19,20,21,
0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19
};
GGML_VQ3_ALIGN64 static const uint8_t ggml_vq3_shuf_sc_gather[64] = {
0, 1, 2,63, 3, 4, 5,63, 6, 7, 8,63, 9,10,11,63,
12,13,14,63, 15,16,17,63, 18,19,20,63, 21,63,20,21,
63,63,63,63, 63,63,63,63, 63,63,63,63, 63,63,63,63,
63,63,63,63, 63,63,63,63, 63,63,63,63, 63,63,63,63
};
GGML_VQ3_ALIGN64 static const uint16_t ggml_vq3_sc_rshift[32] = {
7,7, 7,7, 7,7, 7,7, 7,7, 7,7, 7,7, 7,6,
0,0, 0,0, 0,0, 0,0, 0,0, 0,0, 0,0, 0,0
};
GGML_VQ3_ALIGN64 static const int8_t ggml_vq3_weights_sc[64] = {
1,2,4,0, 1,2,4,0, 1,2,4,0, 1,2,4,0,
1,2,4,0, 1,2,4,0, 1,2,4,0, 1,0,2,4,
0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0,
0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0
};
GGML_VQ3_ALIGN64 static const uint16_t ggml_vq3_shuf_mult_i16[32] = {
0, 0, 0, 0, 2, 2, 2, 2, 4, 4, 4, 4, 6, 6, 6, 6,
8, 8, 8, 8, 10,10,10,10, 12,12,12,12, 14,14,14,14
};
#if defined(__AVX512F__) || (defined(__BMI2__) && (defined(__x86_64__) || defined(_M_X64)))
#include <immintrin.h>
#endif
#if defined(__AVX512F__)
#if defined(__GNUC__) || defined(__clang__)
static inline __m512i ggml_vq3_loadu_si512(const void * p) {
__m512i r;
__asm__("vmovdqu64 %1, %0" : "=v"(r) : "m"(*(const uint64_t *)p));
return r;
}
#else
#define ggml_vq3_loadu_si512(p) _mm512_loadu_si512((const __m512i *)(p))
#endif
#endif
static inline uint64_t ggml_vq3_unpack_scales(uint32_t s24) {
#if defined(__BMI2__) && (defined(__x86_64__) || defined(_M_X64))
return _pdep_u64((uint64_t)s24, 0x0707070707070707ULL) + 0x0202020202020202ULL;
#else
uint64_t m_all = 0x0202020202020202ULL;
for (int b = 0; b < 8; ++b) {
m_all += ((uint64_t)((s24 >> (3 * b)) & 0x7U)) << (8 * b);
}
return m_all;
#endif
}
GGML_VQ3_ALIGN64 static const uint32_t ggml_vq3_shuf_mult512[16] = {
0x80008000, 0x80008000, 0x80018001, 0x80018001,
0x80028002, 0x80028002, 0x80038003, 0x80038003,
0x80048004, 0x80048004, 0x80058005, 0x80058005,
0x80068006, 0x80068006, 0x80078007, 0x80078007
};
GGML_VQ3_ALIGN64 static const uint32_t ggml_q2_lut_abs512[16] = {
0x0104080e, 0x0e080401, 0x0104080e, 0x0e080401,
0x0104080e, 0x0e080401, 0x0104080e, 0x0e080401,
0x0104080e, 0x0e080401, 0x0104080e, 0x0e080401,
0x0104080e, 0x0e080401, 0x0104080e, 0x0e080401
};
GGML_VQ3_ALIGN64 static const uint32_t ggml_q2_shift_qs[16] = {
0, 0, 0, 0, 2, 2, 2, 2, 4, 4, 4, 4, 6, 6, 6, 6
};
GGML_VQ3_ALIGN64 static const uint64_t ggml_q2_ror_qh[8] = {
62, 63, 0, 1, 2, 3, 4, 5
};
#endif // GGML_VQ3_CODEBOOK_H