#ifndef GGML_VQ3_CODEBOOK_H #define GGML_VQ3_CODEBOOK_H #include #define GGML_VQ3_G (8.6373016238e-03f) #if defined(_MSC_VER) #define GGML_VQ3_ALIGN64 __declspec(align(64)) #else #define GGML_VQ3_ALIGN64 __attribute__((aligned(64))) #endif static const int8_t ggml_vq3_cb[128][3] = { { -24, -19, -24 }, { -37, -20, -123 }, { -41, -18, 18 }, { 23, -6, 27 }, { 29, -15, -65 }, { -3, 4, 0 }, { -59, -25, 72 }, { -68, -13, -66 }, { 11, -54, -122 }, { -84, -5, 30 }, { 35, -51, 51 }, { -19, 8, 50 }, { 54, 9, 62 }, { 12, -74, -55 }, { -41, -73, -34 }, { 16, 46, 19 }, { -62, -70, -111 }, { 69, -14, 116 }, { -65, -73, 20 }, { 70, 11, 12 }, { 75, -2, -40 }, { 55, 43, -65 }, { -6, 62, -121 }, { -47, 32, 16 }, { 73, -64, 65 }, { -22, 67, 121 }, { 62, 61, -9 }, { 5, 80, -18 }, { -123, -46, 55 }, { -119, -58, -56 }, { -24, -120, -122 }, { -57, 48, -72 }, { -125, -52, -3 }, { -30, 77, 27 }, { -19, -126, -11 }, { 43, -118, -118 }, { -56, 69, -22 }, { 52, -122, 119 }, { 120, 6, -116 }, { 123, -22, -59 }, { 124, -26, 115 }, { 127, 4, -10 }, { -97, -121, -34 }, { 119, 46, -48 }, { 117, 54, -115 }, { 7, 117, 76 }, { 9, 120, 26 }, { -122, -113, 102 }, { -123, -114, 19 }, { 5, 124, -58 }, { 51, 122, -8 }, { 100, -123, 51 }, { 114, -111, -62 }, { 62, 123, 52 }, { -56, 119, -59 }, { -122, 6, -31 }, { -123, 8, 120 }, { -121, 40, -121 }, { 122, -120, -3 }, { -127, 1, 15 }, { 114, 114, 111 }, { 121, 114, 28 }, { -114, 97, 118 }, { -119, 116, 48 }, { 5, -19, 75 }, { 16, -14, 126 }, { 10, 8, -122 }, { -18, -41, -70 }, { 18, -35, -16 }, { -13, -51, 18 }, { -37, -47, 123 }, { 29, 16, -25 }, { -15, 8, -63 }, { -15, 14, 120 }, { -19, -71, 69 }, { -75, -42, -18 }, { 31, -64, 121 }, { 60, -2, -118 }, { -15, 37, -26 }, { 1, 49, 66 }, { 20, -81, 0 }, { 43, 42, 120 }, { 66, -41, 8 }, { 63, -53, -98 }, { 4, 58, -67 }, { 63, -62, -38 }, { -115, -30, -124 }, { -108, -47, 118 }, { 53, 59, -124 }, { -54, 31, -125 }, { 58, 65, 56 }, { -62, 19, 123 }, { -5, -119, 115 }, { -66, 13, -23 }, { -24, -121, 38 }, { -31, -120, -66 }, { -61, -113, 118 }, { -63, 46, 65 }, { 30, -121, 48 }, { 35, -124, -54 }, { 118, -18, 34 }, { -76, -122, 46 }, { 56, -122, -4 }, { 118, 26, 62 }, { 122, -48, -117 }, { 115, 44, 120 }, { 122, -56, -12 }, { 38, 112, 124 }, { -113, -115, -109 }, { 124, -70, 44 }, { 121, 55, 14 }, { 63, 110, -59 }, { 25, 123, -120 }, { -47, 113, -121 }, { -24, 124, -9 }, { 116, -106, 119 }, { -51, 117, 60 }, { -40, 124, 122 }, { 114, -118, -124 }, { -115, 55, 13 }, { -124, 4, -76 }, { -124, 26, 64 }, { -74, 117, 8 }, { -119, 59, -46 }, { 110, 120, -115 }, { 121, 116, -38 }, { -115, 114, -106 }, { -122, 121, -25 }, }; GGML_VQ3_ALIGN64 static const int8_t ggml_vq3_lut0_lo[64] = { -24, -37, -41, 23, 29, -3, -59, -68, 11, -84, 35, -19, 54, 12, -41, 16, -62, 69, -65, 70, 75, 55, -6, -47, 73, -22, 62, 5, -123, -119, -24, -57, -125, -30, -19, 43, -56, 52, 120, 123, 124, 127, -97, 119, 117, 7, 9, -122, -123, 5, 51, 100, 114, 62, -56, -122, -123, -121, 122, -127, 114, 121, -114, -119 }; GGML_VQ3_ALIGN64 static const int8_t ggml_vq3_lut0_hi[64] = { 5, 16, 10, -18, 18, -13, -37, 29, -15, -15, -19, -75, 31, 60, -15, 1, 20, 43, 66, 63, 4, 63, -115, -108, 53, -54, 58, -62, -5, -66, -24, -31, -61, -63, 30, 35, 118, -76, 56, 118, 122, 115, 122, 38, -113, 124, 121, 63, 25, -47, -24, 116, -51, -40, 114, -115, -124, -124, -74, -119, 110, 121, -115, -122 }; GGML_VQ3_ALIGN64 static const int8_t ggml_vq3_lut1_lo[64] = { -19, -20, -18, -6, -15, 4, -25, -13, -54, -5, -51, 8, 9, -74, -73, 46, -70, -14, -73, 11, -2, 43, 62, 32, -64, 67, 61, 80, -46, -58, -120, 48, -52, 77, -126, -118, 69, -122, 6, -22, -26, 4, -121, 46, 54, 117, 120, -113, -114, 124, 122, -123, -111, 123, 119, 6, 8, 40, -120, 1, 114, 114, 97, 116 }; GGML_VQ3_ALIGN64 static const int8_t ggml_vq3_lut1_hi[64] = { -19, -14, 8, -41, -35, -51, -47, 16, 8, 14, -71, -42, -64, -2, 37, 49, -81, 42, -41, -53, 58, -62, -30, -47, 59, 31, 65, 19, -119, 13, -121, -120, -113, 46, -121, -124, -18, -122, -122, 26, -48, 44, -56, 112, -115, -70, 55, 110, 123, 113, 124, -106, 117, 124, -118, 55, 4, 26, 117, 59, 120, 116, 114, 121 }; GGML_VQ3_ALIGN64 static const int8_t ggml_vq3_lut2_lo[64] = { -24, -123, 18, 27, -65, 0, 72, -66, -122, 30, 51, 50, 62, -55, -34, 19, -111, 116, 20, 12, -40, -65, -121, 16, 65, 121, -9, -18, 55, -56, -122, -72, -3, 27, -11, -118, -22, 119, -116, -59, 115, -10, -34, -48, -115, 76, 26, 102, 19, -58, -8, 51, -62, 52, -59, -31, 120, -121, -3, 15, 111, 28, 118, 48 }; GGML_VQ3_ALIGN64 static const int8_t ggml_vq3_lut2_hi[64] = { 75, 126, -122, -70, -16, 18, 123, -25, -63, 120, 69, -18, 121, -118, -26, 66, 0, 120, 8, -98, -67, -38, -124, 118, -124, -125, 56, 123, 115, -23, 38, -66, 118, 65, 48, -54, 34, 46, -4, 62, -117, 120, -12, 124, -109, 44, 14, -59, -120, -121, -9, 119, 60, 122, -124, 13, -76, 64, 8, -46, -115, -38, -106, -25 }; GGML_VQ3_ALIGN64 static const uint8_t ggml_vq3_perm_rep64[64] = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19,20,21, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19,20,21, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9,10,11,12,13,14,15,16,17,18,19 }; GGML_VQ3_ALIGN64 static const uint8_t ggml_vq3_shuf_sc_gather[64] = { 0, 1, 2,63, 3, 4, 5,63, 6, 7, 8,63, 9,10,11,63, 12,13,14,63, 15,16,17,63, 18,19,20,63, 21,63,20,21, 63,63,63,63, 63,63,63,63, 63,63,63,63, 63,63,63,63, 63,63,63,63, 63,63,63,63, 63,63,63,63, 63,63,63,63 }; GGML_VQ3_ALIGN64 static const uint16_t ggml_vq3_sc_rshift[32] = { 7,7, 7,7, 7,7, 7,7, 7,7, 7,7, 7,7, 7,6, 0,0, 0,0, 0,0, 0,0, 0,0, 0,0, 0,0, 0,0 }; GGML_VQ3_ALIGN64 static const int8_t ggml_vq3_weights_sc[64] = { 1,2,4,0, 1,2,4,0, 1,2,4,0, 1,2,4,0, 1,2,4,0, 1,2,4,0, 1,2,4,0, 1,0,2,4, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0, 0,0,0,0 }; GGML_VQ3_ALIGN64 static const uint16_t ggml_vq3_shuf_mult_i16[32] = { 0, 0, 0, 0, 2, 2, 2, 2, 4, 4, 4, 4, 6, 6, 6, 6, 8, 8, 8, 8, 10,10,10,10, 12,12,12,12, 14,14,14,14 }; #if defined(__AVX512F__) || (defined(__BMI2__) && (defined(__x86_64__) || defined(_M_X64))) #include #endif #if defined(__AVX512F__) #if defined(__GNUC__) || defined(__clang__) static inline __m512i ggml_vq3_loadu_si512(const void * p) { __m512i r; __asm__("vmovdqu64 %1, %0" : "=v"(r) : "m"(*(const uint64_t *)p)); return r; } #else #define ggml_vq3_loadu_si512(p) _mm512_loadu_si512((const __m512i *)(p)) #endif #endif static inline uint64_t ggml_vq3_unpack_scales(uint32_t s24) { #if defined(__BMI2__) && (defined(__x86_64__) || defined(_M_X64)) return _pdep_u64((uint64_t)s24, 0x0707070707070707ULL) + 0x0202020202020202ULL; #else uint64_t m_all = 0x0202020202020202ULL; for (int b = 0; b < 8; ++b) { m_all += ((uint64_t)((s24 >> (3 * b)) & 0x7U)) << (8 * b); } return m_all; #endif } GGML_VQ3_ALIGN64 static const uint32_t ggml_vq3_shuf_mult512[16] = { 0x80008000, 0x80008000, 0x80018001, 0x80018001, 0x80028002, 0x80028002, 0x80038003, 0x80038003, 0x80048004, 0x80048004, 0x80058005, 0x80058005, 0x80068006, 0x80068006, 0x80078007, 0x80078007 }; GGML_VQ3_ALIGN64 static const uint32_t ggml_q2_lut_abs512[16] = { 0x0104080e, 0x0e080401, 0x0104080e, 0x0e080401, 0x0104080e, 0x0e080401, 0x0104080e, 0x0e080401, 0x0104080e, 0x0e080401, 0x0104080e, 0x0e080401, 0x0104080e, 0x0e080401, 0x0104080e, 0x0e080401 }; GGML_VQ3_ALIGN64 static const uint32_t ggml_q2_shift_qs[16] = { 0, 0, 0, 0, 2, 2, 2, 2, 4, 4, 4, 4, 6, 6, 6, 6 }; GGML_VQ3_ALIGN64 static const uint64_t ggml_q2_ror_qh[8] = { 62, 63, 0, 1, 2, 3, 4, 5 }; #endif // GGML_VQ3_CODEBOOK_H