tda45 commited on
Commit
0862f9d
·
verified ·
1 Parent(s): c9c3262

Upload folder using huggingface_hub (part 6)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +3 -0
  2. llama.cpp/LICENSE +21 -0
  3. llama.cpp/Makefile +9 -0
  4. llama.cpp/README.md +604 -0
  5. llama.cpp/SECURITY.md +97 -0
  6. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp +124 -0
  7. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
  8. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/log.comp +17 -0
  9. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
  10. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
  11. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +264 -0
  12. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl +230 -0
  13. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl +35 -0
  14. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +132 -0
  15. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +95 -0
  16. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
  17. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +105 -0
  18. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
  19. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +90 -0
  20. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
  21. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +124 -0
  22. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +156 -0
  23. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +128 -0
  24. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
  25. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +134 -0
  26. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +165 -0
  27. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
  28. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp +141 -0
  29. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl +527 -0
  30. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +466 -0
  31. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +658 -0
  32. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +617 -0
  33. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl +74 -0
  34. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +311 -0
  35. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl +454 -0
  36. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl +93 -0
  37. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/multi_add.comp +194 -0
  38. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
  39. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
  40. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_sgd.comp +22 -0
  41. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +64 -0
  42. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
  43. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +127 -0
  44. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/reglu.comp +9 -0
  45. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
  46. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
  47. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +150 -0
  48. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
  49. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp +65 -0
  50. llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/roll.comp +46 -0
.gitattributes CHANGED
@@ -36,3 +36,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
36
  *.gguf filter=lfs diff=lfs merge=lfs -text
37
  llama.cpp/docs/android/imported-into-android-studio.jpg filter=lfs diff=lfs merge=lfs -text
38
  llama.cpp/docs/development/llama-star/idea-arch.key filter=lfs diff=lfs merge=lfs -text
 
 
 
 
36
  *.gguf filter=lfs diff=lfs merge=lfs -text
37
  llama.cpp/docs/android/imported-into-android-studio.jpg filter=lfs diff=lfs merge=lfs -text
38
  llama.cpp/docs/development/llama-star/idea-arch.key filter=lfs diff=lfs merge=lfs -text
39
+ llama.cpp/media/llama0-banner.png filter=lfs diff=lfs merge=lfs -text
40
+ llama.cpp/media/llama0-logo.png filter=lfs diff=lfs merge=lfs -text
41
+ llama.cpp/media/matmul.png filter=lfs diff=lfs merge=lfs -text
llama.cpp/LICENSE ADDED
@@ -0,0 +1,21 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ MIT License
2
+
3
+ Copyright (c) 2023-2026 The ggml authors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
llama.cpp/Makefile ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ define newline
2
+
3
+
4
+ endef
5
+
6
+ $(error Build system changed:$(newline)\
7
+ The Makefile build has been replaced by CMake.$(newline)$(newline)\
8
+ For build instructions see:$(newline)\
9
+ https://github.com/ggml-org/llama.cpp/blob/master/docs/build.md$(newline)${newline})
llama.cpp/README.md ADDED
@@ -0,0 +1,604 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # llama.cpp
2
+
3
+ ![llama](https://raw.githubusercontent.com/ggml-org/llama.brand/refs/heads/master/cover/llama-cpp/cover-llama-cpp-dark.svg)
4
+
5
+ [![License: MIT](https://img.shields.io/badge/license-MIT-blue.svg)](https://opensource.org/licenses/MIT)
6
+ [![Release](https://img.shields.io/github/v/release/ggml-org/llama.cpp)](https://github.com/ggml-org/llama.cpp/releases)
7
+ [![Server](https://github.com/ggml-org/llama.cpp/actions/workflows/server.yml/badge.svg)](https://github.com/ggml-org/llama.cpp/actions/workflows/server.yml)
8
+ [![Docker](https://github.com/ggml-org/llama.cpp/actions/workflows/docker.yml/badge.svg)](https://github.com/ggml-org/llama.cpp/actions/workflows/docker.yml)
9
+ [![Winget](https://github.com/ggml-org/llama.cpp/actions/workflows/winget.yml/badge.svg)](https://github.com/ggml-org/llama.cpp/actions/workflows/winget.yml)
10
+
11
+ [Manifesto](https://github.com/ggml-org/llama.cpp/discussions/205) / [ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md)
12
+
13
+ LLM inference in C/C++
14
+
15
+ ## Recent API changes
16
+
17
+ - [Changelog for `libllama` API](https://github.com/ggml-org/llama.cpp/issues/9289)
18
+ - [Changelog for `llama-server` REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
19
+
20
+ ## Hot topics
21
+
22
+ - **Hugging Face cache migration: models downloaded with `-hf` are now stored in the standard Hugging Face cache directory, enabling sharing with other HF tools.**
23
+ - **[guide : using the new WebUI of llama.cpp](https://github.com/ggml-org/llama.cpp/discussions/16938)**
24
+ - [guide : running gpt-oss with llama.cpp](https://github.com/ggml-org/llama.cpp/discussions/15396)
25
+ - [[FEEDBACK] Better packaging for llama.cpp to support downstream consumers 🤗](https://github.com/ggml-org/llama.cpp/discussions/15313)
26
+ - Support for the `gpt-oss` model with native MXFP4 format has been added | [PR](https://github.com/ggml-org/llama.cpp/pull/15091) | [Collaboration with NVIDIA](https://blogs.nvidia.com/blog/rtx-ai-garage-openai-oss) | [Comment](https://github.com/ggml-org/llama.cpp/discussions/15095)
27
+ - Multimodal support arrived in `llama-server`: [#12898](https://github.com/ggml-org/llama.cpp/pull/12898) | [documentation](./docs/multimodal.md)
28
+ - VS Code extension for FIM completions: https://github.com/ggml-org/llama.vscode
29
+ - Vim/Neovim plugin for FIM completions: https://github.com/ggml-org/llama.vim
30
+ - Hugging Face Inference Endpoints now support GGUF out of the box! https://github.com/ggml-org/llama.cpp/discussions/9669
31
+ - Hugging Face GGUF editor: [discussion](https://github.com/ggml-org/llama.cpp/discussions/9268) | [tool](https://huggingface.co/spaces/CISCai/gguf-editor)
32
+ - WebGPU support is now available in the browser, see a blog/demo introducing it [here](https://reeselevine.github.io/llamas-on-the-web/).
33
+
34
+ ----
35
+
36
+ ## Quick start
37
+
38
+ Getting started with llama.cpp is straightforward. Here are several ways to install it on your machine:
39
+
40
+ - Install `llama.cpp` using [brew, nix, winget, or conda-forge](docs/install.md)
41
+ - Run with Docker - see our [Docker documentation](docs/docker.md)
42
+ - Download pre-built binaries from the [releases page](https://github.com/ggml-org/llama.cpp/releases)
43
+ - Build from source by cloning this repository - check out [our build guide](docs/build.md)
44
+
45
+ Once installed, you'll need a model to work with. Head to the [Obtaining and quantizing models](#obtaining-and-quantizing-models) section to learn more.
46
+
47
+ Example command:
48
+
49
+ ```sh
50
+ # Use a local model file
51
+ llama-cli -m my_model.gguf
52
+
53
+ # Or download and run a model directly from Hugging Face
54
+ llama-cli -hf ggml-org/gemma-3-1b-it-GGUF
55
+
56
+ # Launch OpenAI-compatible API server
57
+ llama-server -hf ggml-org/gemma-3-1b-it-GGUF
58
+ ```
59
+
60
+ ## Description
61
+
62
+ The main goal of `llama.cpp` is to enable LLM inference with minimal setup and state-of-the-art performance on a wide
63
+ range of hardware - locally and in the cloud.
64
+
65
+ - Plain C/C++ implementation without any dependencies
66
+ - Apple silicon is a first-class citizen - optimized via ARM NEON, Accelerate and Metal frameworks
67
+ - AVX, AVX2, AVX512 and AMX support for x86 architectures
68
+ - RVV, ZVFH, ZFH, ZICBOP and ZIHINTPAUSE support for RISC-V architectures
69
+ - 1.5-bit, 2-bit, 3-bit, 4-bit, 5-bit, 6-bit, and 8-bit integer quantization for faster inference and reduced memory use
70
+ - Custom CUDA kernels for running LLMs on NVIDIA GPUs (support for AMD GPUs via HIP and Moore Threads GPUs via MUSA)
71
+ - Vulkan and SYCL backend support
72
+ - CPU+GPU hybrid inference to partially accelerate models larger than the total VRAM capacity
73
+
74
+ The `llama.cpp` project is the main playground for developing new features for the [ggml](https://github.com/ggml-org/ggml) library.
75
+
76
+ <details>
77
+ <summary>Models</summary>
78
+
79
+ Typically finetunes of the base models below are supported as well.
80
+
81
+ Instructions for adding support for new models: [HOWTO-add-model.md](docs/development/HOWTO-add-model.md)
82
+
83
+ #### Text-only
84
+
85
+ - [X] LLaMA 🦙
86
+ - [x] LLaMA 2 🦙🦙
87
+ - [x] LLaMA 3 🦙🦙🦙
88
+ - [X] [Mistral 7B](https://huggingface.co/mistralai/Mistral-7B-v0.1)
89
+ - [x] [Mixtral MoE](https://huggingface.co/models?search=mistral-ai/Mixtral)
90
+ - [x] [DBRX](https://huggingface.co/databricks/dbrx-instruct)
91
+ - [x] [Jamba](https://huggingface.co/ai21labs)
92
+ - [X] [Falcon](https://huggingface.co/models?search=tiiuae/falcon)
93
+ - [X] [Chinese LLaMA / Alpaca](https://github.com/ymcui/Chinese-LLaMA-Alpaca) and [Chinese LLaMA-2 / Alpaca-2](https://github.com/ymcui/Chinese-LLaMA-Alpaca-2)
94
+ - [X] [Vigogne (French)](https://github.com/bofenghuang/vigogne)
95
+ - [X] [BERT](https://github.com/ggml-org/llama.cpp/pull/5423)
96
+ - [X] [Koala](https://bair.berkeley.edu/blog/2023/04/03/koala/)
97
+ - [X] [Baichuan 1 & 2](https://huggingface.co/models?search=baichuan-inc/Baichuan) + [derivations](https://huggingface.co/hiyouga/baichuan-7b-sft)
98
+ - [X] [Aquila 1 & 2](https://huggingface.co/models?search=BAAI/Aquila)
99
+ - [X] [Starcoder models](https://github.com/ggml-org/llama.cpp/pull/3187)
100
+ - [X] [Refact](https://huggingface.co/smallcloudai/Refact-1_6B-fim)
101
+ - [X] [MPT](https://github.com/ggml-org/llama.cpp/pull/3417)
102
+ - [X] [Bloom](https://github.com/ggml-org/llama.cpp/pull/3553)
103
+ - [x] [Yi models](https://huggingface.co/models?search=01-ai/Yi)
104
+ - [X] [StableLM models](https://huggingface.co/stabilityai)
105
+ - [x] [Deepseek models](https://huggingface.co/models?search=deepseek-ai/deepseek)
106
+ - [x] [Qwen models](https://huggingface.co/models?search=Qwen/Qwen)
107
+ - [x] [PLaMo-13B](https://github.com/ggml-org/llama.cpp/pull/3557)
108
+ - [x] [Phi models](https://huggingface.co/models?search=microsoft/phi)
109
+ - [x] [PhiMoE](https://github.com/ggml-org/llama.cpp/pull/11003)
110
+ - [x] [GPT-2](https://huggingface.co/gpt2)
111
+ - [x] [Orion 14B](https://github.com/ggml-org/llama.cpp/pull/5118)
112
+ - [x] [InternLM2](https://huggingface.co/models?search=internlm2)
113
+ - [x] [CodeShell](https://github.com/WisdomShell/codeshell)
114
+ - [x] [Gemma](https://ai.google.dev/gemma)
115
+ - [x] [Mamba](https://github.com/state-spaces/mamba)
116
+ - [x] [Grok-1](https://huggingface.co/keyfan/grok-1-hf)
117
+ - [x] [Xverse](https://huggingface.co/models?search=xverse)
118
+ - [x] [Command-R models](https://huggingface.co/models?search=CohereForAI/c4ai-command-r)
119
+ - [x] [SEA-LION](https://huggingface.co/models?search=sea-lion)
120
+ - [x] [GritLM-7B](https://huggingface.co/GritLM/GritLM-7B) + [GritLM-8x7B](https://huggingface.co/GritLM/GritLM-8x7B)
121
+ - [x] [OLMo](https://allenai.org/olmo)
122
+ - [x] [OLMo 2](https://allenai.org/olmo)
123
+ - [x] [OLMoE](https://huggingface.co/allenai/OLMoE-1B-7B-0924)
124
+ - [x] [Granite models](https://huggingface.co/collections/ibm-granite/granite-code-models-6624c5cec322e4c148c8b330)
125
+ - [x] [GPT-NeoX](https://github.com/EleutherAI/gpt-neox) + [Pythia](https://github.com/EleutherAI/pythia)
126
+ - [x] [Snowflake-Arctic MoE](https://huggingface.co/collections/Snowflake/arctic-66290090abe542894a5ac520)
127
+ - [x] [Smaug](https://huggingface.co/models?search=Smaug)
128
+ - [x] [Poro 34B](https://huggingface.co/LumiOpen/Poro-34B)
129
+ - [x] [Bitnet b1.58 models](https://huggingface.co/1bitLLM)
130
+ - [x] [Flan T5](https://huggingface.co/models?search=flan-t5)
131
+ - [x] [Open Elm models](https://huggingface.co/collections/apple/openelm-instruct-models-6619ad295d7ae9f868b759ca)
132
+ - [x] [ChatGLM3-6b](https://huggingface.co/THUDM/chatglm3-6b) + [ChatGLM4-9b](https://huggingface.co/THUDM/glm-4-9b) + [GLMEdge-1.5b](https://huggingface.co/THUDM/glm-edge-1.5b-chat) + [GLMEdge-4b](https://huggingface.co/THUDM/glm-edge-4b-chat)
133
+ - [x] [GLM-4-0414](https://huggingface.co/collections/THUDM/glm-4-0414-67f3cbcb34dd9d252707cb2e)
134
+ - [x] [SmolLM](https://huggingface.co/collections/HuggingFaceTB/smollm-6695016cad7167254ce15966)
135
+ - [x] [EXAONE-3.0-7.8B-Instruct](https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct)
136
+ - [x] [FalconMamba Models](https://huggingface.co/collections/tiiuae/falconmamba-7b-66b9a580324dd1598b0f6d4a)
137
+ - [x] [Jais](https://huggingface.co/inceptionai/jais-13b-chat)
138
+ - [x] [Bielik-11B-v2.3](https://huggingface.co/collections/speakleash/bielik-11b-v23-66ee813238d9b526a072408a)
139
+ - [x] [RWKV-7](https://huggingface.co/collections/shoumenchougou/rwkv7-gxx-gguf)
140
+ - [x] [RWKV-6](https://github.com/BlinkDL/RWKV-LM)
141
+ - [x] [QRWKV-6](https://huggingface.co/recursal/QRWKV6-32B-Instruct-Preview-v0.1)
142
+ - [x] [GigaChat-20B-A3B](https://huggingface.co/ai-sage/GigaChat-20B-A3B-instruct)
143
+ - [X] [Trillion-7B-preview](https://huggingface.co/trillionlabs/Trillion-7B-preview)
144
+ - [x] [Ling models](https://huggingface.co/collections/inclusionAI/ling-67c51c85b34a7ea0aba94c32)
145
+ - [x] [Liquid LFM2 models](https://huggingface.co/collections/LiquidAI/lfm2)
146
+ - [x] [Liquid LFM2.5 models](https://huggingface.co/collections/LiquidAI/lfm25)
147
+ - [x] [Liquid Nanos](https://huggingface.co/collections/LiquidAI/liquid-nanos)
148
+ - [x] [Hunyuan models](https://huggingface.co/collections/tencent/hunyuan-dense-model-6890632cda26b19119c9c5e7)
149
+ - [x] [BailingMoeV2 (Ring/Ling 2.0) models](https://huggingface.co/collections/inclusionAI/ling-v2-68bf1dd2fc34c306c1fa6f86)
150
+ - [x] [Mellum models](https://huggingface.co/JetBrains/models?search=mellum)
151
+
152
+ #### Multimodal
153
+
154
+ - [x] [LLaVA 1.5 models](https://huggingface.co/collections/liuhaotian/llava-15-653aac15d994e992e2677a7e), [LLaVA 1.6 models](https://huggingface.co/collections/liuhaotian/llava-16-65b9e40155f60fd046a5ccf2)
155
+ - [x] [BakLLaVA](https://huggingface.co/models?search=SkunkworksAI/Bakllava)
156
+ - [x] [Obsidian](https://huggingface.co/NousResearch/Obsidian-3B-V0.5)
157
+ - [x] [ShareGPT4V](https://huggingface.co/models?search=Lin-Chen/ShareGPT4V)
158
+ - [x] [MobileVLM 1.7B/3B models](https://huggingface.co/models?search=mobileVLM)
159
+ - [x] [Yi-VL](https://huggingface.co/models?search=Yi-VL)
160
+ - [x] [Mini CPM](https://huggingface.co/models?search=MiniCPM)
161
+ - [x] [Moondream](https://huggingface.co/vikhyatk/moondream2)
162
+ - [x] [Bunny](https://github.com/BAAI-DCAI/Bunny)
163
+ - [x] [GLM-EDGE](https://huggingface.co/models?search=glm-edge)
164
+ - [x] [Qwen2-VL](https://huggingface.co/collections/Qwen/qwen2-vl-66cee7455501d7126940800d)
165
+ - [x] [LFM2-VL](https://huggingface.co/collections/LiquidAI/lfm2-vl-68963bbc84a610f7638d5ffa)
166
+
167
+ </details>
168
+
169
+ <details>
170
+ <summary>Bindings</summary>
171
+
172
+ - Python: [ddh0/easy-llama](https://github.com/ddh0/easy-llama)
173
+ - Python: [abetlen/llama-cpp-python](https://github.com/abetlen/llama-cpp-python)
174
+ - Go: [go-skynet/go-llama.cpp](https://github.com/go-skynet/go-llama.cpp)
175
+ - Node.js: [withcatai/node-llama-cpp](https://github.com/withcatai/node-llama-cpp)
176
+ - JS/TS (llama.cpp server client): [lgrammel/modelfusion](https://modelfusion.dev/integration/model-provider/llamacpp)
177
+ - JS/TS (Programmable Prompt Engine CLI): [offline-ai/cli](https://github.com/offline-ai/cli)
178
+ - JavaScript/Wasm (works in browser): [tangledgroup/llama-cpp-wasm](https://github.com/tangledgroup/llama-cpp-wasm)
179
+ - Typescript/Wasm (nicer API, available on npm): [ngxson/wllama](https://github.com/ngxson/wllama)
180
+ - Ruby: [yoshoku/llama_cpp.rb](https://github.com/yoshoku/llama_cpp.rb)
181
+ - Ruby: [docusealco/rllama](https://github.com/docusealco/rllama)
182
+ - Rust (more features): [edgenai/llama_cpp-rs](https://github.com/edgenai/llama_cpp-rs)
183
+ - Rust (nicer API): [mdrokz/rust-llama.cpp](https://github.com/mdrokz/rust-llama.cpp)
184
+ - Rust (more direct bindings): [utilityai/llama-cpp-rs](https://github.com/utilityai/llama-cpp-rs)
185
+ - Rust (automated build from crates.io): [ShelbyJenkins/llm_client](https://github.com/ShelbyJenkins/llm_client)
186
+ - C#/.NET: [SciSharp/LLamaSharp](https://github.com/SciSharp/LLamaSharp)
187
+ - C#/VB.NET (more features - community license): [LM-Kit.NET](https://docs.lm-kit.com/lm-kit-net/index.html)
188
+ - Scala 3: [donderom/llm4s](https://github.com/donderom/llm4s)
189
+ - Clojure: [phronmophobic/llama.clj](https://github.com/phronmophobic/llama.clj)
190
+ - React Native: [mybigday/llama.rn](https://github.com/mybigday/llama.rn)
191
+ - Java: [kherud/java-llama.cpp](https://github.com/kherud/java-llama.cpp)
192
+ - Java: [QuasarByte/llama-cpp-jna](https://github.com/QuasarByte/llama-cpp-jna)
193
+ - Zig: [deins/llama.cpp.zig](https://github.com/Deins/llama.cpp.zig)
194
+ - Flutter/Dart: [netdur/llama_cpp_dart](https://github.com/netdur/llama_cpp_dart)
195
+ - Flutter: [xuegao-tzx/Fllama](https://github.com/xuegao-tzx/Fllama)
196
+ - PHP (API bindings and features built on top of llama.cpp): [distantmagic/resonance](https://github.com/distantmagic/resonance) [(more info)](https://github.com/ggml-org/llama.cpp/pull/6326)
197
+ - Guile Scheme: [guile_llama_cpp](https://savannah.nongnu.org/projects/guile-llama-cpp)
198
+ - Swift [srgtuszy/llama-cpp-swift](https://github.com/srgtuszy/llama-cpp-swift)
199
+ - Swift [ShenghaiWang/SwiftLlama](https://github.com/ShenghaiWang/SwiftLlama)
200
+ - Delphi [Embarcadero/llama-cpp-delphi](https://github.com/Embarcadero/llama-cpp-delphi)
201
+ - Go (no CGo needed): [hybridgroup/yzma](https://github.com/hybridgroup/yzma)
202
+ - Android: [llama.android](/examples/llama.android)
203
+
204
+ </details>
205
+
206
+ <details>
207
+ <summary>UIs</summary>
208
+
209
+ *(to have a project listed here, it should clearly state that it depends on `llama.cpp`)*
210
+
211
+ - [AI Sublime Text plugin](https://github.com/yaroslavyaroslav/OpenAI-sublime-text) (MIT)
212
+ - [BonzAI App](https://apps.apple.com/us/app/bonzai-your-local-ai-agent/id6752847988) (proprietary)
213
+ - [cztomsik/ava](https://github.com/cztomsik/ava) (MIT)
214
+ - [Dot](https://github.com/alexpinel/Dot) (GPL)
215
+ - [eva](https://github.com/ylsdamxssjxxdd/eva) (MIT)
216
+ - [iohub/collama](https://github.com/iohub/coLLaMA) (Apache-2.0)
217
+ - [janhq/jan](https://github.com/janhq/jan) (AGPL)
218
+ - [johnbean393/Sidekick](https://github.com/johnbean393/Sidekick) (MIT)
219
+ - [KanTV](https://github.com/zhouwg/kantv?tab=readme-ov-file) (Apache-2.0)
220
+ - [KodiBot](https://github.com/firatkiral/kodibot) (GPL)
221
+ - [llama.vim](https://github.com/ggml-org/llama.vim) (MIT)
222
+ - [LARS](https://github.com/abgulati/LARS) (AGPL)
223
+ - [Llama Assistant](https://github.com/vietanhdev/llama-assistant) (GPL)
224
+ - [LlamaLib](https://github.com/undreamai/LlamaLib) (Apache-2.0)
225
+ - [LLMFarm](https://github.com/guinmoon/LLMFarm?tab=readme-ov-file) (MIT)
226
+ - [LLMUnity](https://github.com/undreamai/LLMUnity) (MIT)
227
+ - [LMStudio](https://lmstudio.ai/) (proprietary)
228
+ - [LocalAI](https://github.com/mudler/LocalAI) (MIT)
229
+ - [LostRuins/koboldcpp](https://github.com/LostRuins/koboldcpp) (AGPL)
230
+ - [MindMac](https://mindmac.app) (proprietary)
231
+ - [MindWorkAI/AI-Studio](https://github.com/MindWorkAI/AI-Studio) (FSL-1.1-MIT)
232
+ - [Mobile-Artificial-Intelligence/maid](https://github.com/Mobile-Artificial-Intelligence/maid) (MIT)
233
+ - [Mozilla-Ocho/llamafile](https://github.com/Mozilla-Ocho/llamafile) (Apache-2.0)
234
+ - [nat/openplayground](https://github.com/nat/openplayground) (MIT)
235
+ - [nomic-ai/gpt4all](https://github.com/nomic-ai/gpt4all) (MIT)
236
+ - [ollama/ollama](https://github.com/ollama/ollama) (MIT)
237
+ - [oobabooga/text-generation-webui](https://github.com/oobabooga/text-generation-webui) (AGPL)
238
+ - [PocketPal AI](https://github.com/a-ghorbani/pocketpal-ai) (MIT)
239
+ - [psugihara/FreeChat](https://github.com/psugihara/FreeChat) (MIT)
240
+ - [ptsochantaris/emeltal](https://github.com/ptsochantaris/emeltal) (MIT)
241
+ - [pythops/tenere](https://github.com/pythops/tenere) (AGPL)
242
+ - [ramalama](https://github.com/containers/ramalama) (MIT)
243
+ - [semperai/amica](https://github.com/semperai/amica) (MIT)
244
+ - [withcatai/catai](https://github.com/withcatai/catai) (MIT)
245
+ - [Autopen](https://github.com/blackhole89/autopen) (GPL)
246
+
247
+ </details>
248
+
249
+ <details>
250
+ <summary>Tools</summary>
251
+
252
+ - [akx/ggify](https://github.com/akx/ggify) – download PyTorch models from Hugging Face Hub and convert them to GGML
253
+ - [akx/ollama-dl](https://github.com/akx/ollama-dl) – download models from the Ollama library to be used directly with llama.cpp
254
+ - [crashr/gppm](https://github.com/crashr/gppm) – launch llama.cpp instances utilizing NVIDIA Tesla P40 or P100 GPUs with reduced idle power consumption
255
+ - [gpustack/gguf-parser](https://github.com/gpustack/gguf-parser-go/tree/main/cmd/gguf-parser) - review/check the GGUF file and estimate the memory usage
256
+ - [Styled Lines](https://marketplace.unity.com/packages/tools/generative-ai/styled-lines-llama-cpp-model-292902) (proprietary licensed, async wrapper of inference part for game development in Unity3d with pre-built Mobile and Web platform wrappers and a model example)
257
+ - [unslothai/unsloth](https://github.com/unslothai/unsloth) – 🦥 exports/saves fine-tuned and trained models to GGUF (Apache-2.0)
258
+
259
+ </details>
260
+
261
+ <details>
262
+ <summary>Infrastructure</summary>
263
+
264
+ - [Paddler](https://github.com/intentee/paddler) - Open-source LLMOps platform for hosting and scaling AI in your own infrastructure
265
+ - [GPUStack](https://github.com/gpustack/gpustack) - Manage GPU clusters for running LLMs
266
+ - [llama_cpp_canister](https://github.com/onicai/llama_cpp_canister) - llama.cpp as a smart contract on the Internet Computer, using WebAssembly
267
+ - [llama-swap](https://github.com/mostlygeek/llama-swap) - transparent proxy that adds automatic model switching with llama-server
268
+ - [Kalavai](https://github.com/kalavai-net/kalavai-client) - Crowdsource end to end LLM deployment at any scale
269
+ - [llmaz](https://github.com/InftyAI/llmaz) - ☸️ Easy, advanced inference platform for large language models on Kubernetes.
270
+ - [LLMKube](https://github.com/defilantech/llmkube) - Kubernetes operator for llama.cpp with multi-GPU and Apple Silicon Metal
271
+ support"
272
+ </details>
273
+
274
+ <details>
275
+ <summary>Games</summary>
276
+
277
+ - [Lucy's Labyrinth](https://github.com/MorganRO8/Lucys_Labyrinth) - A simple maze game where agents controlled by an AI model will try to trick you.
278
+
279
+ </details>
280
+
281
+
282
+ ## Supported backends
283
+
284
+ | Backend | Target devices |
285
+ | --- | --- |
286
+ | [Metal](docs/build.md#metal-build) | Apple Silicon |
287
+ | [BLAS](docs/build.md#blas-build) | All |
288
+ | [BLIS](docs/backend/BLIS.md) | All |
289
+ | [SYCL](docs/backend/SYCL.md) | Intel GPU |
290
+ | [OpenVINO [In Progress]](docs/backend/OPENVINO.md) | Intel CPUs, GPUs, and NPUs |
291
+ | [MUSA](docs/build.md#musa) | Moore Threads GPU |
292
+ | [CUDA](docs/build.md#cuda) | Nvidia GPU |
293
+ | [HIP](docs/build.md#hip) | AMD GPU |
294
+ | [ZenDNN](docs/build.md#zendnn) | AMD CPU |
295
+ | [Vulkan](docs/build.md#vulkan) | GPU |
296
+ | [CANN](docs/build.md#cann) | Ascend NPU |
297
+ | [OpenCL](docs/backend/OPENCL.md) | Adreno GPU |
298
+ | [IBM zDNN](docs/backend/zDNN.md) | IBM Z & LinuxONE |
299
+ | [WebGPU](docs/build.md#webgpu) | All |
300
+ | [RPC](https://github.com/ggml-org/llama.cpp/tree/master/tools/rpc) | All |
301
+ | [Hexagon [In Progress]](docs/backend/snapdragon/README.md) | Snapdragon |
302
+ | [VirtGPU](docs/backend/VirtGPU.md) | VirtGPU APIR |
303
+
304
+ ## Obtaining and quantizing models
305
+
306
+ The [Hugging Face](https://huggingface.co) platform hosts a [number of LLMs](https://huggingface.co/models?library=gguf&sort=trending) compatible with `llama.cpp`:
307
+
308
+ - [Trending](https://huggingface.co/models?library=gguf&sort=trending)
309
+ - [LLaMA](https://huggingface.co/models?sort=trending&search=llama+gguf)
310
+
311
+ You can either manually download the GGUF file or directly use any `llama.cpp`-compatible models from [Hugging Face](https://huggingface.co/) or other model hosting sites, by using this CLI argument: `-hf <user>/<model>[:quant]`. For example:
312
+
313
+ ```sh
314
+ llama-cli -hf ggml-org/gemma-3-1b-it-GGUF
315
+ ```
316
+
317
+ By default, the CLI would download from Hugging Face, you can switch to other options with the environment variable `MODEL_ENDPOINT`. The `MODEL_ENDPOINT` must point to a Hugging Face compatible API endpoint.
318
+
319
+ After downloading a model, use the CLI tools to run it locally - see below.
320
+
321
+ `llama.cpp` requires the model to be stored in the [GGUF](https://github.com/ggml-org/ggml/blob/master/docs/gguf.md) file format. Models in other data formats can be converted to GGUF using the `convert_*.py` Python scripts in this repo.
322
+
323
+ The Hugging Face platform provides a variety of online tools for converting, quantizing and hosting models with `llama.cpp`:
324
+
325
+ - Use the [GGUF-my-repo space](https://huggingface.co/spaces/ggml-org/gguf-my-repo) to convert to GGUF format and quantize model weights to smaller sizes
326
+ - Use the [GGUF-my-LoRA space](https://huggingface.co/spaces/ggml-org/gguf-my-lora) to convert LoRA adapters to GGUF format (more info: https://github.com/ggml-org/llama.cpp/discussions/10123)
327
+ - Use the [GGUF-editor space](https://huggingface.co/spaces/CISCai/gguf-editor) to edit GGUF meta data in the browser (more info: https://github.com/ggml-org/llama.cpp/discussions/9268)
328
+ - Use the [Inference Endpoints](https://ui.endpoints.huggingface.co/) to directly host `llama.cpp` in the cloud (more info: https://github.com/ggml-org/llama.cpp/discussions/9669)
329
+
330
+ To learn more about model quantization, [read this documentation](tools/quantize/README.md)
331
+
332
+ ## [`llama-cli`](tools/cli)
333
+
334
+ #### A CLI tool for accessing and experimenting with most of `llama.cpp`'s functionality.
335
+
336
+ - <details open>
337
+ <summary>Run in conversation mode</summary>
338
+
339
+ Models with a built-in chat template will automatically activate conversation mode. If this doesn't occur, you can manually enable it by adding `-cnv` and specifying a suitable chat template with `--chat-template NAME`
340
+
341
+ ```bash
342
+ llama-cli -m model.gguf
343
+
344
+ # > hi, who are you?
345
+ # Hi there! I'm your helpful assistant! I'm an AI-powered chatbot designed to assist and provide information to users like you. I'm here to help answer your questions, provide guidance, and offer support on a wide range of topics. I'm a friendly and knowledgeable AI, and I'm always happy to help with anything you need. What's on your mind, and how can I assist you today?
346
+ #
347
+ # > what is 1+1?
348
+ # Easy peasy! The answer to 1+1 is... 2!
349
+ ```
350
+
351
+ </details>
352
+
353
+ - <details>
354
+ <summary>Run in conversation mode with custom chat template</summary>
355
+
356
+ ```bash
357
+ # use the "chatml" template (use -h to see the list of supported templates)
358
+ llama-cli -m model.gguf -cnv --chat-template chatml
359
+
360
+ # use a custom template
361
+ llama-cli -m model.gguf -cnv --in-prefix 'User: ' --reverse-prompt 'User:'
362
+ ```
363
+
364
+ </details>
365
+
366
+ - <details>
367
+ <summary>Constrain the output with a custom grammar</summary>
368
+
369
+ ```bash
370
+ llama-cli -m model.gguf -n 256 --grammar-file grammars/json.gbnf -p 'Request: schedule a call at 8pm; Command:'
371
+
372
+ # {"appointmentTime": "8pm", "appointmentDetails": "schedule a a call"}
373
+ ```
374
+
375
+ The [grammars/](grammars/) folder contains a handful of sample grammars. To write your own, check out the [GBNF Guide](grammars/README.md).
376
+
377
+ For authoring more complex JSON grammars, check out https://grammar.intrinsiclabs.ai/
378
+
379
+ </details>
380
+
381
+
382
+ ## [`llama-server`](tools/server)
383
+
384
+ #### A lightweight, [OpenAI API](https://github.com/openai/openai-openapi) compatible, HTTP server for serving LLMs.
385
+
386
+ - <details open>
387
+ <summary>Start a local HTTP server with default configuration on port 8080</summary>
388
+
389
+ ```bash
390
+ llama-server -m model.gguf --port 8080
391
+
392
+ # Basic web UI can be accessed via browser: http://localhost:8080
393
+ # Chat completion endpoint: http://localhost:8080/v1/chat/completions
394
+ ```
395
+
396
+ </details>
397
+
398
+ - <details>
399
+ <summary>Support multiple-users and parallel decoding</summary>
400
+
401
+ ```bash
402
+ # up to 4 concurrent requests, each with 4096 max context
403
+ llama-server -m model.gguf -c 16384 -np 4
404
+ ```
405
+
406
+ </details>
407
+
408
+ - <details>
409
+ <summary>Enable speculative decoding</summary>
410
+
411
+ ```bash
412
+ # the draft.gguf model should be a small variant of the target model.gguf
413
+ llama-server -m model.gguf -md draft.gguf
414
+ ```
415
+
416
+ </details>
417
+
418
+ - <details>
419
+ <summary>Serve an embedding model</summary>
420
+
421
+ ```bash
422
+ # use the /embedding endpoint
423
+ llama-server -m model.gguf --embedding --pooling cls -ub 8192
424
+ ```
425
+
426
+ </details>
427
+
428
+ - <details>
429
+ <summary>Serve a reranking model</summary>
430
+
431
+ ```bash
432
+ # use the /reranking endpoint
433
+ llama-server -m model.gguf --reranking
434
+ ```
435
+
436
+ </details>
437
+
438
+ - <details>
439
+ <summary>Constrain all outputs with a grammar</summary>
440
+
441
+ ```bash
442
+ # custom grammar
443
+ llama-server -m model.gguf --grammar-file grammar.gbnf
444
+
445
+ # JSON
446
+ llama-server -m model.gguf --grammar-file grammars/json.gbnf
447
+ ```
448
+
449
+ </details>
450
+
451
+
452
+ ## [`llama-perplexity`](tools/perplexity)
453
+
454
+ #### A tool for measuring the [perplexity](tools/perplexity/README.md) [^1] (and other quality metrics) of a model over a given text.
455
+
456
+ - <details open>
457
+ <summary>Measure the perplexity over a text file</summary>
458
+
459
+ ```bash
460
+ llama-perplexity -m model.gguf -f file.txt
461
+
462
+ # [1]15.2701,[2]5.4007,[3]5.3073,[4]6.2965,[5]5.8940,[6]5.6096,[7]5.7942,[8]4.9297, ...
463
+ # Final estimate: PPL = 5.4007 +/- 0.67339
464
+ ```
465
+
466
+ </details>
467
+
468
+ - <details>
469
+ <summary>Measure KL divergence</summary>
470
+
471
+ ```bash
472
+ # TODO
473
+ ```
474
+
475
+ </details>
476
+
477
+ [^1]: [https://huggingface.co/docs/transformers/perplexity](https://huggingface.co/docs/transformers/perplexity)
478
+
479
+ ## [`llama-bench`](tools/llama-bench)
480
+
481
+ #### Benchmark the performance of the inference for various parameters.
482
+
483
+ - <details open>
484
+ <summary>Run default benchmark</summary>
485
+
486
+ ```bash
487
+ llama-bench -m model.gguf
488
+
489
+ # Output:
490
+ # | model | size | params | backend | threads | test | t/s |
491
+ # | ------------------- | ---------: | ---------: | ---------- | ------: | ------------: | -------------------: |
492
+ # | qwen2 1.5B Q4_0 | 885.97 MiB | 1.54 B | Metal,BLAS | 16 | pp512 | 5765.41 ± 20.55 |
493
+ # | qwen2 1.5B Q4_0 | 885.97 MiB | 1.54 B | Metal,BLAS | 16 | tg128 | 197.71 ± 0.81 |
494
+ #
495
+ # build: 3e0ba0e60 (4229)
496
+ ```
497
+
498
+ </details>
499
+
500
+ ## [`llama-simple`](examples/simple)
501
+
502
+ #### A minimal example for implementing apps with `llama.cpp`. Useful for developers.
503
+
504
+ - <details>
505
+ <summary>Basic text completion</summary>
506
+
507
+ ```bash
508
+ llama-simple -m model.gguf
509
+
510
+ # Hello my name is Kaitlyn and I am a 16 year old girl. I am a junior in high school and I am currently taking a class called "The Art of
511
+ ```
512
+
513
+ </details>
514
+
515
+
516
+ ## Contributing
517
+
518
+ - Contributors can open PRs
519
+ - Collaborators will be invited based on contributions
520
+ - Maintainers can push to branches in the `llama.cpp` repo and merge PRs into the `master` branch
521
+ - Any help with managing issues, PRs and projects is very appreciated!
522
+ - See [good first issues](https://github.com/ggml-org/llama.cpp/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) for tasks suitable for first contributions
523
+ - Read the [CONTRIBUTING.md](CONTRIBUTING.md) for more information
524
+ - Make sure to read this: [Inference at the edge](https://github.com/ggml-org/llama.cpp/discussions/205)
525
+ - A bit of backstory for those who are interested: [Changelog podcast](https://changelog.com/podcast/532)
526
+
527
+ ## Other documentation
528
+
529
+ - [cli](tools/cli/README.md)
530
+ - [completion](tools/completion/README.md)
531
+ - [server](tools/server/README.md)
532
+ - [GBNF grammars](grammars/README.md)
533
+
534
+ #### Development documentation
535
+
536
+ - [How to build](docs/build.md)
537
+ - [Running on Docker](docs/docker.md)
538
+ - [Build on Android](docs/android.md)
539
+ - [Multi-GPU usage](docs/multi-gpu.md)
540
+ - [Performance troubleshooting](docs/development/token_generation_performance_tips.md)
541
+ - [GGML tips & tricks](https://github.com/ggml-org/llama.cpp/wiki/GGML-Tips-&-Tricks)
542
+
543
+ #### Seminal papers and background on the models
544
+
545
+ If your issue is with model generation quality, then please at least scan the following links and papers to understand the limitations of LLaMA models. This is especially important when choosing an appropriate model size and appreciating both the significant and subtle differences between LLaMA models and ChatGPT:
546
+ - LLaMA:
547
+ - [Introducing LLaMA: A foundational, 65-billion-parameter large language model](https://ai.facebook.com/blog/large-language-model-llama-meta-ai/)
548
+ - [LLaMA: Open and Efficient Foundation Language Models](https://arxiv.org/abs/2302.13971)
549
+ - GPT-3
550
+ - [Language Models are Few-Shot Learners](https://arxiv.org/abs/2005.14165)
551
+ - GPT-3.5 / InstructGPT / ChatGPT:
552
+ - [Aligning language models to follow instructions](https://openai.com/research/instruction-following)
553
+ - [Training language models to follow instructions with human feedback](https://arxiv.org/abs/2203.02155)
554
+
555
+ ## XCFramework
556
+ The XCFramework is a precompiled version of the library for iOS, visionOS, tvOS,
557
+ and macOS. It can be used in Swift projects without the need to compile the
558
+ library from source. For example:
559
+ ```swift
560
+ // swift-tools-version: 5.10
561
+ // The swift-tools-version declares the minimum version of Swift required to build this package.
562
+
563
+ import PackageDescription
564
+
565
+ let package = Package(
566
+ name: "MyLlamaPackage",
567
+ targets: [
568
+ .executableTarget(
569
+ name: "MyLlamaPackage",
570
+ dependencies: [
571
+ "LlamaFramework"
572
+ ]),
573
+ .binaryTarget(
574
+ name: "LlamaFramework",
575
+ url: "https://github.com/ggml-org/llama.cpp/releases/download/b5046/llama-b5046-xcframework.zip",
576
+ checksum: "c19be78b5f00d8d29a25da41042cb7afa094cbf6280a225abe614b03b20029ab"
577
+ )
578
+ ]
579
+ )
580
+ ```
581
+ The above example is using an intermediate build `b5046` of the library. This can be modified
582
+ to use a different version by changing the URL and checksum.
583
+
584
+ ## Completions
585
+ Command-line completion is available for some environments.
586
+
587
+ #### Bash Completion
588
+ ```bash
589
+ $ build/bin/llama-cli --completion-bash > ~/.llama-completion.bash
590
+ $ source ~/.llama-completion.bash
591
+ ```
592
+ Optionally this can be added to your `.bashrc` or `.bash_profile` to load it
593
+ automatically. For example:
594
+ ```console
595
+ $ echo "source ~/.llama-completion.bash" >> ~/.bashrc
596
+ ```
597
+
598
+ ## Dependencies
599
+
600
+ - [yhirose/cpp-httplib](https://github.com/yhirose/cpp-httplib) - Single-header HTTP server, used by `llama-server` - MIT license
601
+ - [stb-image](https://github.com/nothings/stb) - Single-header image format decoder, used by multimodal subsystem - Public domain
602
+ - [nlohmann/json](https://github.com/nlohmann/json) - Single-header JSON library, used by various tools/examples - MIT License
603
+ - [miniaudio.h](https://github.com/mackron/miniaudio) - Single-header audio format decoder, used by multimodal subsystem - Public domain
604
+ - [subprocess.h](https://github.com/sheredom/subprocess.h) - Single-header process launching solution for C and C++ - Public domain
llama.cpp/SECURITY.md ADDED
@@ -0,0 +1,97 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Security Policy
2
+
3
+ - [**Reporting a vulnerability**](#reporting-a-vulnerability)
4
+ - [**Requirements**](#requirements)
5
+ - [**Covered Topics**](#covered-topics)
6
+ - [**Using llama.cpp securely**](#using-llamacpp-securely)
7
+ - [Untrusted models](#untrusted-models)
8
+ - [Untrusted inputs](#untrusted-inputs)
9
+ - [Data privacy](#data-privacy)
10
+ - [Untrusted environments or networks](#untrusted-environments-or-networks)
11
+ - [Multi-Tenant environments](#multi-tenant-environments)
12
+
13
+ ## Reporting a vulnerability
14
+
15
+ > [!IMPORTANT]
16
+ > The private security disclosure program is disabled until further notice. Please submit patches with fixes directly to the repo as public PRs. Emails will be ignored.
17
+
18
+ If you have discovered a security vulnerability in this project that falls inside the [covered topics](#covered-topics), please report it privately. **Do not disclose it as a public issue.** This gives us time to work with you to fix the issue before public exposure, reducing the chance that the exploit will be used before a patch is released.
19
+
20
+ Please disclose it as a private [security advisory](https://github.com/ggml-org/llama.cpp/security/advisories/new).
21
+
22
+ A team of volunteers on a reasonable-effort basis maintains this project. As such, please give us at least 90 days to work on a fix before public exposure.
23
+
24
+ ### Requirements
25
+
26
+ Before submitting your report, ensure you meet the following requirements:
27
+
28
+ - You have read this policy and fully understand it.
29
+ - AI is only permitted in an assistive capacity as stated in [AGENTS.md](AGENTS.md). We do not accept reports that are written exclusively by AI.
30
+ - Your report must include a working Proof-of-Concept in the form of a script and/or attached files.
31
+
32
+ Maintainers reserve the right to close the report if these requirements are not fulfilled.
33
+
34
+ ### Covered Topics
35
+
36
+ Only vulnerabilities that fall within these parts of the project are considered valid. For problems falling outside of this list, please report them as issues.
37
+
38
+ - `src/**/*`
39
+ - `ggml/**/*`
40
+ - `gguf-py/**/*`
41
+ - `tools/server/*`, **excluding** the following topics:
42
+ - Web UI
43
+ - Features marked as experimental
44
+ - Features not recommended for use in untrusted environments (e.g., router, MCP)
45
+ - Bugs that can lead to Denial-of-Service attack
46
+
47
+ Note that none of the topics under [Using llama.cpp securely](#using-llamacpp-securely) are considered vulnerabilities in LLaMA C++.
48
+
49
+ For vulnerabilities that fall within the `vendor` directory, please report them directly to the third-party project.
50
+
51
+ ## Using llama.cpp securely
52
+
53
+ ### Untrusted models
54
+ Be careful when running untrusted models. This classification includes models created by unknown developers or utilizing data obtained from unknown sources.
55
+
56
+ *Always execute untrusted models within a secure, isolated environment such as a sandbox* (e.g., containers, virtual machines). This helps protect your system from potentially malicious code.
57
+
58
+ > [!NOTE]
59
+ > The trustworthiness of a model is not binary. You must always determine the proper level of caution depending on the specific model and how it matches your use case and risk tolerance.
60
+
61
+ ### Untrusted inputs
62
+
63
+ Some models accept various input formats (text, images, audio, etc.). The libraries converting these inputs have varying security levels, so it's crucial to isolate the model and carefully pre-process inputs to mitigate script injection risks.
64
+
65
+ For maximum security when handling untrusted inputs, you may need to employ the following:
66
+
67
+ * Sandboxing: Isolate the environment where the inference happens.
68
+ * Pre-analysis: Check how the model performs by default when exposed to prompt injection (e.g. using [fuzzing for prompt injection](https://github.com/FonduAI/awesome-prompt-injection?tab=readme-ov-file#tools)). This will give you leads on how hard you will have to work on the next topics.
69
+ * Updates: Keep both LLaMA C++ and your libraries updated with the latest security patches.
70
+ * Input Sanitation: Before feeding data to the model, sanitize inputs rigorously. This involves techniques such as:
71
+ * Validation: Enforce strict rules on allowed characters and data types.
72
+ * Filtering: Remove potentially malicious scripts or code fragments.
73
+ * Encoding: Convert special characters into safe representations.
74
+ * Verification: Run tooling that identifies potential script injections (e.g. [models that detect prompt injection attempts](https://python.langchain.com/docs/guides/safety/hugging_face_prompt_injection)).
75
+
76
+ ### Data privacy
77
+
78
+ To protect sensitive data from potential leaks or unauthorized access, it is crucial to sandbox the model execution. This means running the model in a secure, isolated environment, which helps mitigate many attack vectors.
79
+
80
+ ### Untrusted environments or networks
81
+
82
+ If you can't run your models in a secure and isolated environment or if it must be exposed to an untrusted network, make sure to take the following security precautions:
83
+ * Do not use the RPC backend, [ggml-rpc-server](https://github.com/ggml-org/llama.cpp/tree/master/tools/rpc) and [llama-server](https://github.com/ggml-org/llama.cpp/tree/master/tools/server) functionality (see https://github.com/ggml-org/llama.cpp/pull/13061).
84
+ * Confirm the hash of any downloaded artifact (e.g. pre-trained model weights) matches a known-good value.
85
+ * Encrypt your data if sending it over the network.
86
+
87
+ ### Multi-Tenant environments
88
+
89
+ If you intend to run multiple models in parallel with shared memory, it is your responsibility to ensure the models do not interact or access each other's data. The primary areas of concern are tenant isolation, resource allocation, model sharing and hardware attacks.
90
+
91
+ 1. Tenant Isolation: Models should run separately with strong isolation methods to prevent unwanted data access. Separating networks is crucial for isolation, as it prevents unauthorized access to data or models and malicious users from sending graphs to execute under another tenant's identity.
92
+
93
+ 2. Resource Allocation: A denial of service caused by one model can impact the overall system health. Implement safeguards like rate limits, access controls, and health monitoring.
94
+
95
+ 3. Model Sharing: In a multitenant model sharing design, tenants and users must understand the security risks of running code provided by others. Since there are no reliable methods to detect malicious models, sandboxing the model execution is the recommended approach to mitigate the risk.
96
+
97
+ 4. Hardware Attacks: GPUs or TPUs can also be attacked. [Researches](https://scholar.google.com/scholar?q=gpu+side+channel) has shown that side channel attacks on GPUs are possible, which can make data leak from other models or processes running on the same system at the same time.
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp ADDED
@@ -0,0 +1,124 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+
3
+ #extension GL_EXT_shader_16bit_storage : require
4
+ #extension GL_EXT_control_flow_attributes : require
5
+ #extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
6
+
7
+ #include "types.glsl"
8
+
9
+ layout (push_constant) uniform parameter
10
+ {
11
+ BDA_STORAGE_T dst_addr;
12
+ uint32_t nb10;
13
+ uint32_t nb11;
14
+ uint32_t nb12;
15
+ uint32_t nb13;
16
+ uint32_t s0;
17
+ uint32_t s1;
18
+ uint32_t s2;
19
+ uint32_t p0;
20
+ uint32_t p1;
21
+ uint32_t p2;
22
+ uint32_t d0;
23
+ uint32_t d1;
24
+ uint32_t d2;
25
+ uint32_t IW;
26
+ uint32_t IH;
27
+ uint32_t ID;
28
+ uint32_t IC;
29
+ uint32_t KW;
30
+ uint32_t OH;
31
+ uint32_t KD_KH_KW;
32
+ uint32_t KH_KW;
33
+ uint32_t IC_KD_KH_KW;
34
+ uint32_t N_OD_OH;
35
+ uint32_t OD_OH;
36
+ uint32_t OD_OH_OW_IC_KD_KH_KW;
37
+ uint32_t OH_OW_IC_KD_KH_KW;
38
+ uint32_t OW_IC_KD_KH_KW;
39
+ uint32_t misalign_offsets;
40
+ } p;
41
+
42
+ uint get_aoffset() { return p.misalign_offsets >> 16; }
43
+ uint get_doffset() { return p.misalign_offsets & 0xFFFF; }
44
+
45
+ layout(constant_id = 0) const uint BLOCK_SIZE = 32;
46
+
47
+ layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
48
+
49
+ layout (binding = 0) readonly buffer X {A_TYPE data_a[];};
50
+ layout (binding = 1) writeonly buffer D {D_TYPE data_d[];};
51
+
52
+ #if BDA
53
+ layout (buffer_reference) buffer D_ptr {D_TYPE d;};
54
+ #endif
55
+
56
+ void main() {
57
+ const uint32_t i = gl_GlobalInvocationID.x;
58
+
59
+ uint32_t nb10 = p.nb10;
60
+ uint32_t nb11 = p.nb11;
61
+ uint32_t nb12 = p.nb12;
62
+ uint32_t nb13 = p.nb13;
63
+ uint32_t s0 = p.s0;
64
+ uint32_t s1 = p.s1;
65
+ uint32_t s2 = p.s2;
66
+ uint32_t p0 = p.p0;
67
+ uint32_t p1 = p.p1;
68
+ uint32_t p2 = p.p2;
69
+ uint32_t d0 = p.d0;
70
+ uint32_t d1 = p.d1;
71
+ uint32_t d2 = p.d2;
72
+ uint32_t IW = p.IW;
73
+ uint32_t IH = p.IH;
74
+ uint32_t ID = p.ID;
75
+ uint32_t IC = p.IC;
76
+ uint32_t KW = p.KW;
77
+ uint32_t OH = p.OH;
78
+ uint32_t KD_KH_KW = p.KD_KH_KW;
79
+ uint32_t KH_KW = p.KH_KW;
80
+ uint32_t IC_KD_KH_KW = p.IC_KD_KH_KW;
81
+ uint32_t N_OD_OH = p.N_OD_OH;
82
+ uint32_t OD_OH = p.OD_OH;
83
+ uint32_t OD_OH_OW_IC_KD_KH_KW = p.OD_OH_OW_IC_KD_KH_KW;
84
+ uint32_t OH_OW_IC_KD_KH_KW = p.OH_OW_IC_KD_KH_KW;
85
+ uint32_t OW_IC_KD_KH_KW = p.OW_IC_KD_KH_KW;
86
+
87
+ if (i >= IC_KD_KH_KW) {
88
+ return;
89
+ }
90
+
91
+ const uint32_t iic = i / KD_KH_KW;
92
+ const uint32_t ikd = (i - iic * KD_KH_KW) / KH_KW;
93
+ const uint32_t ikh = (i - iic * KD_KH_KW - ikd * KH_KW) / KW;
94
+ const uint32_t ikw = i % KW;
95
+
96
+ const uint32_t iow = gl_GlobalInvocationID.y;
97
+ for (uint32_t iz = gl_GlobalInvocationID.z; iz < N_OD_OH; iz += gl_NumWorkGroups.z) {
98
+ const uint32_t in_ = iz / OD_OH;
99
+ const uint32_t iod = (iz - in_*OD_OH) / OH;
100
+ const uint32_t ioh = iz % OH;
101
+
102
+ const uint32_t iiw = iow * s0 + ikw * d0 - p0;
103
+ const uint32_t iih = ioh * s1 + ikh * d1 - p1;
104
+ const uint32_t iid = iod * s2 + ikd * d2 - p2;
105
+
106
+ const BDA_OFFSET_T offset_dst = BDA_OFFSET_T(in_)*OD_OH_OW_IC_KD_KH_KW + BDA_OFFSET_T(iod)*OH_OW_IC_KD_KH_KW + BDA_OFFSET_T(ioh)*OW_IC_KD_KH_KW + BDA_OFFSET_T(iow)*IC_KD_KH_KW + iic*KD_KH_KW + ikd * KH_KW + ikh*KW + ikw;
107
+
108
+ const uint32_t offset_src = (in_*IC + iic)*nb13 + iid*nb12 + iih*nb11 + iiw*nb10;
109
+ #if BDA
110
+ D_ptr dst_addr = D_ptr(p.dst_addr + D_SIZE * offset_dst);
111
+ if (iih >= IH || iiw >= IW || iid >= ID) {
112
+ dst_addr.d = D_TYPE(0.0f);
113
+ } else {
114
+ dst_addr.d = D_TYPE(data_a[offset_src + get_aoffset()]);
115
+ }
116
+ #else
117
+ if (iih >= IH || iiw >= IW || iid >= ID) {
118
+ data_d[offset_dst + get_doffset()] = D_TYPE(0.0f);
119
+ } else {
120
+ data_d[offset_dst + get_doffset()] = D_TYPE(data_a[offset_src + get_aoffset()]);
121
+ }
122
+ #endif
123
+ }
124
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+
3
+ #include "generic_unary_head.glsl"
4
+ #include "types.glsl"
5
+
6
+ #extension GL_EXT_control_flow_attributes : enable
7
+ #define BLOCK_SIZE 512
8
+
9
+ layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in;
10
+
11
+ shared FLOAT_TYPE sum[BLOCK_SIZE];
12
+
13
+ void main() {
14
+ const uint row = gl_WorkGroupID.z * 262144 + gl_WorkGroupID.y * 512 + gl_WorkGroupID.x;
15
+ const uint tid = gl_LocalInvocationID.x;
16
+
17
+ const uint a_base = get_aoffset() + src0_idx(row * p.ne00);
18
+ const uint d_base = get_doffset() + dst_idx(row * p.ne10);
19
+
20
+ sum[tid] = FLOAT_TYPE(0.0f); // partial sum for thread in warp
21
+
22
+ [[unroll]] for (uint i0 = tid; i0 < p.ne00; i0 += BLOCK_SIZE) {
23
+ const FLOAT_TYPE xi = FLOAT_TYPE(data_a[a_base + i0*p.nb00]);
24
+ sum[tid] += xi * xi;
25
+ }
26
+
27
+ // sum up partial sums and write back result
28
+ barrier();
29
+ [[unroll]] for (int s = BLOCK_SIZE / 2; s > 0; s >>= 1) {
30
+ if (tid < s) {
31
+ sum[tid] += sum[tid + s];
32
+ }
33
+ barrier();
34
+ }
35
+
36
+ const FLOAT_TYPE scale = 1.0f / max(sqrt(sum[0]), FLOAT_TYPE(p.param1));
37
+
38
+ [[unroll]] for (uint i0 = tid; i0 < p.ne00; i0 += BLOCK_SIZE) {
39
+ data_d[d_base + i0*p.nb10] = D_TYPE(scale * FLOAT_TYPE(data_a[a_base + i0*p.nb00]));
40
+ }
41
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/log.comp ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+
3
+ #include "types.glsl"
4
+ #include "generic_unary_head.glsl"
5
+
6
+ layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
7
+
8
+ void main() {
9
+ const uint idx = get_idx();
10
+
11
+ if (idx >= p.ne) {
12
+ return;
13
+ }
14
+
15
+ const float val = float(data_a[get_aoffset() + src0_idx(idx)]);
16
+ data_d[get_doffset() + dst_idx(idx)] = D_TYPE(log(val));
17
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp ADDED
@@ -0,0 +1,27 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+
3
+ #include "types.glsl"
4
+ #include "generic_binary_head.glsl"
5
+
6
+ const uint num_threads = 256;
7
+
8
+ layout(local_size_x = num_threads, local_size_y = 1, local_size_z = 1) in;
9
+
10
+ void main() {
11
+ uint idx = get_idx();
12
+
13
+ // num_threads * num_iter must equal 512, to match the wg_denoms and get_idx calculation
14
+ const uint num_iter = 2;
15
+
16
+ [[unroll]] for (uint i = 0; i < num_iter; ++i) {
17
+ if (idx >= p.ne) {
18
+ continue;
19
+ }
20
+ uint i00, i01, i02, i03;
21
+ get_indices(idx, i00, i01, i02, i03);
22
+
23
+ data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = D_TYPE(FLOAT_TYPE(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]) * FLOAT_TYPE(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]));
24
+
25
+ idx += num_threads;
26
+ }
27
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+
3
+ #extension GL_EXT_control_flow_attributes : enable
4
+
5
+ layout(local_size_x = 256, local_size_y = 1, local_size_z = 1) in;
6
+
7
+ layout (binding = 0) readonly buffer A {float data_a[];};
8
+ layout (binding = 0) readonly buffer A4 {vec4 data_a4[];};
9
+ layout (binding = 1) writeonly buffer D {float data_d[];};
10
+ layout (binding = 1) writeonly buffer D4 {vec4 data_d4[];};
11
+
12
+ layout (push_constant) uniform parameter {
13
+ uint ne;
14
+ uint k_num;
15
+ } p;
16
+
17
+ void main() {
18
+ // Each invocation handles four consecutive components
19
+ const uint idx = gl_GlobalInvocationID.x * 4;
20
+
21
+ if (idx >= p.ne) {
22
+ return;
23
+ }
24
+
25
+ // Check if all four components are in bounds and aligned,
26
+ // then use vector loads
27
+ if (idx + 3 < p.ne && (p.ne % 4) == 0) {
28
+ vec4 result = vec4(0.0f);
29
+
30
+ [[unroll]] for (uint i = 0; i < p.k_num; i++) {
31
+ result += data_a4[(i * p.ne + idx) / 4];
32
+ }
33
+
34
+ data_d4[idx / 4] = result;
35
+ } else {
36
+ [[unroll]] for (uint j = 0; j < 4; ++j) {
37
+ if (idx + j < p.ne) {
38
+ float result = 0.0f;
39
+
40
+ [[unroll]] for (uint i = 0; i < p.k_num; i++) {
41
+ result += data_a[i * p.ne + idx + j];
42
+ }
43
+
44
+ data_d[idx + j] = result;
45
+ }
46
+ }
47
+ }
48
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp ADDED
@@ -0,0 +1,264 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+
3
+ #extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
4
+
5
+ #include "mul_mat_vec_base.glsl"
6
+ #include "dequant_funcs.glsl"
7
+
8
+ layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
9
+
10
+ #if !defined(DATA_A_F32) && !defined(DATA_A_F16) && !defined(DATA_A_BF16)
11
+ #define K_PER_ITER 8
12
+ #else
13
+ #define K_PER_ITER 4
14
+ #endif
15
+
16
+
17
+ uint a_offset, b_offset, d_offset, y_offset;
18
+
19
+ vec4 load_b(const uint j, const uint iybs, const uint iqs, const bool lastiter, out bool OOB_y, out bool OOB_z, out bool OOB_w) {
20
+ // Check if the latter elements are OOB, and don't fetch B or accumulate it.
21
+ OOB_y = lastiter && (iybs + iqs + y_offset >= p.ncols);
22
+ OOB_z = lastiter && (iybs + iqs + y_offset*2 >= p.ncols);
23
+ OOB_w = lastiter && (iybs + iqs + y_offset*3 >= p.ncols);
24
+
25
+ if (!OOB_w) {
26
+ return vec4(FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs]),
27
+ FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs + y_offset]),
28
+ FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs + y_offset*2]),
29
+ FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs + y_offset*3]));
30
+ } else if (!OOB_z) {
31
+ return vec4(FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs]),
32
+ FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs + y_offset]),
33
+ FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs + y_offset*2]),
34
+ 0);
35
+ } else if (!OOB_y) {
36
+ return vec4(FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs]),
37
+ FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs + y_offset]),
38
+ 0, 0);
39
+ } else {
40
+ return vec4(FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs]),
41
+ 0, 0, 0);
42
+ }
43
+ }
44
+
45
+ void iter(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const uint first_row, const uint num_rows, const uint tid, const uint i, bool lastiter)
46
+ {
47
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
48
+ const uint col = i*BLOCK_SIZE + K_PER_ITER*tid;
49
+ const uint iqs = (col%QUANT_K)/QUANT_R; // quant index
50
+ const uint iybs = col - col%QUANT_K; // y block start index
51
+
52
+ #if K_PER_ITER == 8
53
+ #if QUANT_R == 2
54
+ // Note that we end up fetching bogus elements here, but its fine as they'll be
55
+ // within an accessible block.
56
+ const vec4 bv02 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + iybs + iqs) / 4]);
57
+ const vec4 bv13 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + iybs + iqs + y_offset) / 4]);
58
+ const vec4 bv0 = vec4(bv02.x, bv13.x, bv02.y, bv13.y);
59
+ const vec4 bv1 = vec4(bv02.z, bv13.z, bv02.w, bv13.w);
60
+ #else
61
+ const vec4 bv0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + iybs + iqs) / 4]);
62
+ const vec4 bv1 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + iybs + iqs) / 4 + 1]);
63
+ #endif
64
+ #else
65
+ bool OOB_y;
66
+ bool OOB_z;
67
+ bool OOB_w;
68
+
69
+ const vec4 b = load_b(j, iybs, iqs, lastiter, OOB_y, OOB_z, OOB_w);
70
+ #endif
71
+ uint ibi = first_row*p.ncols;
72
+ [[unroll]] for (uint n = 0; n < num_rows; ++n) {
73
+ const uint ib = (ibi + col)/QUANT_K; // block index
74
+ ibi += p.ncols;
75
+
76
+ #if K_PER_ITER == 8
77
+ vec4 v = dequantize4(ib, iqs, a_offset);
78
+ vec4 v2 = dequantize4(ib, iqs+(4/QUANT_R), a_offset);
79
+
80
+ const vec2 dm = get_dm(ib, a_offset);
81
+ if (dm.y != 0) { // quant has min component
82
+ v = v * dm.x + dm.y;
83
+ v2 = v2 * dm.x + dm.y;
84
+ }
85
+
86
+ // matrix multiplication
87
+ FLOAT_TYPE rowtmp = dot(bv0, v);
88
+ rowtmp += dot(bv1, v2);
89
+
90
+ if (dm.y == 0)
91
+ rowtmp *= dm.x;
92
+
93
+ temp[j][n] += rowtmp;
94
+ #else
95
+ if (!OOB_w) {
96
+ const vec4 v = dequantize4(ib, iqs, a_offset);
97
+ temp[j][n] += dot(v, b);
98
+ } else if (!OOB_z) {
99
+ const vec2 v0 = dequantize(ib, iqs, a_offset);
100
+ const FLOAT_TYPE v1 = dequantize1(ib + 2/QUANT_R, iqs, a_offset);
101
+ const vec3 v = vec3(v0.x, v0.y, v1);
102
+ const vec3 b0 = vec3(b.x, b.y, b.z);
103
+ temp[j][n] += dot(v, b0);
104
+ } else if (!OOB_y) {
105
+ const vec2 v0 = dequantize(ib, iqs, a_offset);
106
+ const vec2 b0 = vec2(b.x, b.y);
107
+ temp[j][n] += dot(v0, b0);
108
+ } else {
109
+ const FLOAT_TYPE v = dequantize1(ib, iqs, a_offset);
110
+ temp[j][n] = fma(v, b.x, temp[j][n]);
111
+ }
112
+ #endif
113
+ }
114
+ }
115
+ }
116
+
117
+ #if defined(DATA_A_F32) || defined(DATA_A_F16) || defined(DATA_A_BF16)
118
+ void iter_aligned_nonquant(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const uint first_row, const uint num_rows, const uint tid, const uint i)
119
+ {
120
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
121
+ const uint col = i*BLOCK_SIZE + K_PER_ITER*tid;
122
+ const uint iqs = 0; // quant index
123
+ const uint iybs = col; // y block start index
124
+
125
+ const vec4 b = data_b_v4[(j*p.batch_stride_b + b_offset + iybs + iqs) / 4];
126
+
127
+ uint ibi = first_row*p.ncols;
128
+ [[unroll]] for (uint n = 0; n < num_rows; ++n) {
129
+ const uint ib = (ibi + col)/QUANT_K; // block index
130
+ ibi += p.ncols;
131
+
132
+ const vec4 v = dequantize4_2aligned(ib, iqs, a_offset);
133
+
134
+ // matrix multiplication
135
+ temp[j][n] += dot(v, b);
136
+ }
137
+ }
138
+ }
139
+ #endif
140
+
141
+ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
142
+ const uint tid = gl_LocalInvocationID.x;
143
+
144
+ get_offsets(a_offset, b_offset, d_offset);
145
+ const bool is_aligned_nonquant =
146
+ p.batch_stride_b % 4 == 0 && b_offset % 4 == 0 &&
147
+ p.ncols % 4 == 0 && BLOCK_SIZE % 4 == 0 &&
148
+ K_PER_ITER == 4;
149
+
150
+ y_offset = QUANT_R == 1 ? 1 : QUANT_K/2;
151
+
152
+ FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];
153
+
154
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
155
+ [[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) {
156
+ temp[j][i] = FLOAT_TYPE(0);
157
+ }
158
+ }
159
+
160
+ uint num_iters = p.ncols / (K_PER_ITER * BLOCK_SIZE);
161
+ if (num_iters * K_PER_ITER * BLOCK_SIZE + K_PER_ITER*tid < p.ncols) {
162
+ num_iters++;
163
+ }
164
+ int unroll_count = 4;
165
+ uint unrolled_iters = num_iters & ~(unroll_count - 1);
166
+
167
+ uint i = 0;
168
+
169
+ #if K_PER_ITER == 4
170
+ // If the K dimension is odd, we need lastiter==true on the last iteration
171
+ // so OOB is computed correctly. Skip some unrolling to make that happen.
172
+ if ((p.ncols & 3) != 0 &&
173
+ unrolled_iters == num_iters &&
174
+ unrolled_iters > 0) {
175
+ unrolled_iters -= unroll_count;
176
+ }
177
+ if (is_aligned_nonquant) {
178
+ while (i < unrolled_iters) {
179
+ // Manually partially unroll the loop
180
+ [[unroll]] for (uint k = 0; k < unroll_count; ++k) {
181
+ iter_aligned_nonquant(temp, first_row, num_rows, tid, i*K_PER_ITER);
182
+ i++;
183
+ }
184
+ }
185
+ } else {
186
+ #endif
187
+ while (i < unrolled_iters) {
188
+ // Manually partially unroll the loop
189
+ [[unroll]] for (uint k = 0; k < unroll_count; ++k) {
190
+ iter(temp, first_row, num_rows, tid, i*K_PER_ITER, false);
191
+ i++;
192
+ }
193
+ }
194
+ #if K_PER_ITER == 4
195
+ }
196
+ #endif
197
+
198
+ unroll_count = 2;
199
+ unrolled_iters = num_iters & ~(unroll_count - 1);
200
+
201
+ #if K_PER_ITER == 4
202
+ if ((p.ncols & 3) != 0 &&
203
+ unrolled_iters == num_iters &&
204
+ unrolled_iters > 0) {
205
+ unrolled_iters -= unroll_count;
206
+ }
207
+
208
+ if (is_aligned_nonquant) {
209
+ while (i < unrolled_iters && is_aligned_nonquant) {
210
+ // Manually partially unroll the loop
211
+ [[unroll]] for (uint k = 0; k < unroll_count; ++k) {
212
+ iter_aligned_nonquant(temp, first_row, num_rows, tid, i*K_PER_ITER);
213
+ i++;
214
+ }
215
+ }
216
+ } else {
217
+ #endif
218
+ while (i < unrolled_iters) {
219
+ // Manually partially unroll the loop
220
+ [[unroll]] for (uint k = 0; k < unroll_count; ++k) {
221
+ iter(temp, first_row, num_rows, tid, i*K_PER_ITER, false);
222
+ i++;
223
+ }
224
+ }
225
+ #if K_PER_ITER == 4
226
+ }
227
+ #endif
228
+
229
+ #if K_PER_ITER == 4
230
+ if (is_aligned_nonquant) {
231
+ while (i < num_iters) {
232
+ iter_aligned_nonquant(temp, first_row, num_rows, tid, i*K_PER_ITER);
233
+ i++;
234
+ }
235
+ } else {
236
+ #endif
237
+ while (i < num_iters) {
238
+ iter(temp, first_row, num_rows, tid, i*K_PER_ITER, true);
239
+ i++;
240
+ }
241
+ #if K_PER_ITER == 4
242
+ }
243
+ #endif
244
+
245
+ reduce_result(temp, d_offset, first_row, num_rows, tid);
246
+ }
247
+
248
+ void main() {
249
+ const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z);
250
+
251
+ #ifdef NEEDS_INIT_IQ_SHMEM
252
+ init_iq_shmem(gl_WorkGroupSize);
253
+ #endif
254
+
255
+ // do NUM_ROWS at a time, unless there aren't enough remaining rows
256
+ if (first_row + NUM_ROWS <= p.stride_d) {
257
+ compute_outputs(first_row, NUM_ROWS);
258
+ } else {
259
+ if (first_row >= p.stride_d) {
260
+ return;
261
+ }
262
+ compute_outputs(first_row, min(NUM_ROWS, p.stride_d - first_row));
263
+ }
264
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl ADDED
@@ -0,0 +1,230 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #extension GL_EXT_control_flow_attributes : enable
2
+ #extension GL_EXT_shader_16bit_storage : require
3
+ #extension GL_EXT_shader_8bit_storage : require
4
+
5
+ #if USE_SUBGROUP_ADD || USE_SUBGROUP_ADD_NO_SHMEM
6
+ #extension GL_KHR_shader_subgroup_basic : require
7
+ #extension GL_KHR_shader_subgroup_arithmetic : require
8
+ #endif
9
+
10
+ #ifdef MUL_MAT_ID
11
+ #define EXPERT_COUNT 8
12
+ #endif
13
+
14
+ #include "mul_mat_vec_iface.glsl"
15
+
16
+ layout (push_constant) uniform parameter
17
+ {
18
+ uint ncols;
19
+ uint stride_a;
20
+ uint stride_b;
21
+ uint stride_d;
22
+
23
+ uint batch_stride_a;
24
+ uint batch_stride_b;
25
+ uint batch_stride_d;
26
+
27
+ uint fusion_flags;
28
+
29
+ #ifdef MUL_MAT_ID
30
+ uint nei0;
31
+ uint ne11;
32
+ uint expert_i1;
33
+ uint nbi1;
34
+ #else
35
+ uint base_work_group_y;
36
+ uint ne02;
37
+ uint ne12;
38
+ uint broadcast2;
39
+ uint broadcast3;
40
+ #endif
41
+ } p;
42
+
43
+ #ifdef MUL_MAT_ID
44
+ uint expert_id;
45
+ #endif
46
+
47
+ void get_offsets(out uint a_offset, out uint b_offset, out uint d_offset) {
48
+ #ifdef MUL_MAT_ID
49
+ const uint expert_i0 = gl_WorkGroupID.y;
50
+ #else
51
+ const uint batch_idx = gl_WorkGroupID.y + p.base_work_group_y;
52
+ #endif
53
+
54
+ #ifndef MUL_MAT_ID
55
+ uint batch_idx_a = 0;
56
+ if (batch_idx != 0) {
57
+ const uint i13 = batch_idx / p.ne12;
58
+ const uint i12 = batch_idx % p.ne12;
59
+
60
+ const uint i03 = i13 / p.broadcast3;
61
+ const uint i02 = i12 / p.broadcast2;
62
+
63
+ batch_idx_a = i03 * p.ne02 + i02;
64
+ }
65
+ #else
66
+ expert_id = data_ids[expert_i0 + p.expert_i1 * p.nbi1];
67
+ #endif
68
+
69
+ a_offset =
70
+ #ifdef MUL_MAT_ID
71
+ expert_id * (p.batch_stride_a / QUANT_K);
72
+ #else
73
+ batch_idx_a * (p.batch_stride_a / QUANT_K);
74
+ #endif
75
+ b_offset =
76
+ #ifdef MUL_MAT_ID
77
+ (expert_i0 % p.ne11) * p.stride_b + p.expert_i1 * p.batch_stride_b;
78
+ #else
79
+ batch_idx * p.batch_stride_b;
80
+ #endif
81
+ d_offset =
82
+ #ifdef MUL_MAT_ID
83
+ expert_i0 * p.stride_d + p.expert_i1 * p.batch_stride_d;
84
+ #else
85
+ batch_idx * p.batch_stride_d;
86
+ #endif
87
+ }
88
+
89
+ layout (constant_id = 0) const uint BLOCK_SIZE = 32;
90
+ layout (constant_id = 1) const uint NUM_ROWS = 1;
91
+ layout (constant_id = 2) const uint NUM_COLS = 1;
92
+
93
+ #ifdef USE_SUBGROUP_ADD_NO_SHMEM
94
+ void reduce_result(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const in uint32_t d_offset, const in uint32_t first_row, const in uint32_t num_rows, const in uint32_t tid) {
95
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
96
+ [[unroll]] for (uint n = 0; n < num_rows; ++n) {
97
+ temp[j][n] = subgroupAdd(temp[j][n]);
98
+ }
99
+ }
100
+
101
+ if (tid == 0) {
102
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
103
+ [[unroll]] for (uint n = 0; n < num_rows; ++n) {
104
+ #ifdef MUL_MAT_ID
105
+ if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0) {
106
+ temp[j][n] += FLOAT_TYPE(data_fuse0[expert_id*p.stride_d + first_row + n]);
107
+ }
108
+ if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE0) != 0) {
109
+ const uint expert_i0 = gl_GlobalInvocationID.y;
110
+ temp[j][n] *= FLOAT_TYPE(data_fuse0[expert_i0]);
111
+ }
112
+ if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE1) != 0) {
113
+ const uint expert_i0 = gl_GlobalInvocationID.y;
114
+ temp[j][n] *= FLOAT_TYPE(data_fuse1[expert_i0]);
115
+ }
116
+ #else
117
+ if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0) {
118
+ temp[j][n] += FLOAT_TYPE(data_fuse0[j*p.batch_stride_d + d_offset + first_row + n]);
119
+ }
120
+ if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS1) != 0) {
121
+ temp[j][n] += FLOAT_TYPE(data_fuse1[j*p.batch_stride_d + d_offset + first_row + n]);
122
+ }
123
+ #endif
124
+ data_d[j*p.batch_stride_d + d_offset + first_row + n] = D_TYPE(temp[j][n]);
125
+ }
126
+ }
127
+ }
128
+ }
129
+ #else
130
+ shared FLOAT_TYPE tmpsh[NUM_COLS][NUM_ROWS][BLOCK_SIZE];
131
+
132
+ void reduce_result(FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const in uint32_t d_offset, const in uint32_t first_row, const in uint32_t num_rows, const in uint32_t tid) {
133
+ // subgroupAdd is probably faster on devices that support it,
134
+ // particularly when the workgroup has more than one subgroup
135
+ #if USE_SUBGROUP_ADD
136
+ // sum up partial sums within a subgroup
137
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
138
+ [[unroll]] for (uint n = 0; n < num_rows; ++n) {
139
+ temp[j][n] = subgroupAdd(temp[j][n]);
140
+ }
141
+ }
142
+
143
+ // Go through shared memory to sum partials across subgroups
144
+ if (gl_SubgroupInvocationID == 0) {
145
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
146
+ [[unroll]] for (uint n = 0; n < num_rows; ++n) {
147
+ tmpsh[j][n][gl_SubgroupID] = temp[j][n];
148
+ }
149
+ }
150
+ }
151
+ barrier();
152
+ if (tid == 0) {
153
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
154
+ [[unroll]] for (uint n = 0; n < num_rows; ++n) {
155
+ temp[j][n] = FLOAT_TYPE(0);
156
+ [[unroll]] for (uint s = 0; s < gl_NumSubgroups; ++s) {
157
+ temp[j][n] += tmpsh[j][n][s];
158
+ }
159
+ #ifdef MUL_MAT_ID
160
+ if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0) {
161
+ temp[j][n] += FLOAT_TYPE(data_fuse0[expert_id*p.stride_d + first_row + n]);
162
+ }
163
+ if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE0) != 0) {
164
+ const uint expert_i0 = gl_GlobalInvocationID.y;
165
+ temp[j][n] *= FLOAT_TYPE(data_fuse0[expert_i0]);
166
+ }
167
+ if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE1) != 0) {
168
+ const uint expert_i0 = gl_GlobalInvocationID.y;
169
+ temp[j][n] *= FLOAT_TYPE(data_fuse1[expert_i0]);
170
+ }
171
+ #else
172
+ if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0) {
173
+ temp[j][n] += FLOAT_TYPE(data_fuse0[j*p.batch_stride_d + d_offset + first_row + n]);
174
+ }
175
+ if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS1) != 0) {
176
+ temp[j][n] += FLOAT_TYPE(data_fuse1[j*p.batch_stride_d + d_offset + first_row + n]);
177
+ }
178
+ #endif
179
+ data_d[j*p.batch_stride_d + d_offset + first_row + n] = D_TYPE(temp[j][n]);
180
+ }
181
+ }
182
+ }
183
+ #else
184
+ // sum up partial sums and write back result
185
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
186
+ [[unroll]] for (uint n = 0; n < num_rows; ++n) {
187
+ tmpsh[j][n][tid] = temp[j][n];
188
+ }
189
+ }
190
+ barrier();
191
+ [[unroll]] for (uint s = BLOCK_SIZE/2; s > 0; s >>= 1) {
192
+ if (tid < s) {
193
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
194
+ [[unroll]] for (uint n = 0; n < num_rows; ++n) {
195
+ tmpsh[j][n][tid] += tmpsh[j][n][tid + s];
196
+ }
197
+ }
198
+ }
199
+ barrier();
200
+ }
201
+ if (tid == 0) {
202
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
203
+ [[unroll]] for (uint n = 0; n < num_rows; ++n) {
204
+ #ifdef MUL_MAT_ID
205
+ if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0) {
206
+ tmpsh[j][n][0] += FLOAT_TYPE(data_fuse0[expert_id*p.stride_d + first_row + n]);
207
+ }
208
+ if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE0) != 0) {
209
+ const uint expert_i0 = gl_GlobalInvocationID.y;
210
+ tmpsh[j][n][0] *= FLOAT_TYPE(data_fuse0[expert_i0]);
211
+ }
212
+ if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE1) != 0) {
213
+ const uint expert_i0 = gl_GlobalInvocationID.y;
214
+ tmpsh[j][n][0] *= FLOAT_TYPE(data_fuse1[expert_i0]);
215
+ }
216
+ #else
217
+ if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0) {
218
+ tmpsh[j][n][0] += FLOAT_TYPE(data_fuse0[j*p.batch_stride_d + d_offset + first_row + n]);
219
+ }
220
+ if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS1) != 0) {
221
+ tmpsh[j][n][0] += FLOAT_TYPE(data_fuse1[j*p.batch_stride_d + d_offset + first_row + n]);
222
+ }
223
+ #endif
224
+ data_d[j*p.batch_stride_d + d_offset + first_row + n] = D_TYPE(tmpsh[j][n][0]);
225
+ }
226
+ }
227
+ }
228
+ #endif
229
+ }
230
+ #endif
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #include "types.glsl"
2
+
3
+ #define MAT_VEC_FUSION_FLAGS_BIAS0 0x1
4
+ #define MAT_VEC_FUSION_FLAGS_BIAS1 0x2
5
+ #define MAT_VEC_FUSION_FLAGS_SCALE0 0x4
6
+ #define MAT_VEC_FUSION_FLAGS_SCALE1 0x8
7
+
8
+ layout (binding = 0) readonly buffer A {A_TYPE data_a[];};
9
+ #if defined(A_TYPEV4)
10
+ layout (binding = 0) readonly buffer AV4 {A_TYPEV4 data_a_v4[];};
11
+ #endif
12
+ #if defined(A_TYPE_PACKED16)
13
+ layout (binding = 0) readonly buffer A_PACKED16 {A_TYPE_PACKED16 data_a_packed16[];};
14
+ #endif
15
+ #if defined(A_TYPE_PACKED32)
16
+ layout (binding = 0) readonly buffer A_PACKED32 {A_TYPE_PACKED32 data_a_packed32[];};
17
+ #endif
18
+
19
+ layout (binding = 1) readonly buffer B {B_TYPE data_b[];};
20
+ #ifdef B_TYPEV2
21
+ layout (binding = 1) readonly buffer BV2 {B_TYPEV2 data_b_v2[];};
22
+ #endif
23
+ #ifdef B_TYPEV4
24
+ layout (binding = 1) readonly buffer BV4 {B_TYPEV4 data_b_v4[];};
25
+ #endif
26
+
27
+ layout (binding = 2) writeonly buffer D {D_TYPE data_d[];};
28
+
29
+ layout (binding = 3) readonly buffer Fuse0 {D_TYPE data_fuse0[];};
30
+ layout (binding = 4) readonly buffer Fuse1 {D_TYPE data_fuse1[];};
31
+
32
+ #ifdef MUL_MAT_ID
33
+ layout (binding = 5) readonly buffer IDS {int data_ids[];};
34
+ #endif
35
+
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp ADDED
@@ -0,0 +1,132 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+ #extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
3
+
4
+ #include "mul_mat_vec_base.glsl"
5
+
6
+ layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
7
+
8
+ FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];
9
+
10
+ void calc_superblock(const uint a_offset, const uint b_offset, const uint ib32, const uint i,
11
+ const uint num_blocks_per_row, const uint first_row, const uint num_rows) {
12
+ // Compute starting index in matrix B for this superblock
13
+ const uint y_idx = i * QUANT_K + 32 * ib32;
14
+ uint ibi = a_offset + first_row * num_blocks_per_row + i;
15
+
16
+ // Precompute indices for quantization lookup tables
17
+ const uint qh_base = 2 * ib32;
18
+ const uint qs_base = 4 * ib32;
19
+ const uint sc_index = ib32 / 2;
20
+ const uint sc_shift = 6 * (ib32 & 1);
21
+
22
+ // Loop over rows in the superblock
23
+ [[unroll]] for (uint n = 0; n < num_rows; ++n) {
24
+ // Load per-block scales and shift for quantization
25
+ const uint16_t[4] scales = data_a[ibi].scales;
26
+ const u16vec4 s = u16vec4(scales[0], scales[1], scales[2], scales[3]) >> 12;
27
+ const float d = float(unpackHalf2x16(s.x | (s.y << 4) | (s.z << 8) | (s.w << 12)).x);
28
+ const uint sc = data_a[ibi].scales[sc_index] >> sc_shift;
29
+
30
+ // Temporary caches for decoding
31
+ FLOAT_TYPE dl_cache[4];
32
+ uint16_t gvf_cache[4];
33
+ float delta_cache[4];
34
+
35
+ // Precompute the multiplier and lookup values for 4 sub-blocks
36
+ [[unroll]] for (uint l = 0; l < 4; ++l) {
37
+ dl_cache[l] = FLOAT_TYPE(d * (2 * bitfieldExtract(sc, 3 * int(l / 2), 3) + 1));
38
+ const uint qh = data_a[ibi].qh[qh_base + l / 2] >> (4 * (l & 1));
39
+ const uint qs = data_a[ibi].qs[qs_base + l];
40
+ gvf_cache[l] = iq1s_grid[qs | ((qh & 7) << 8)];
41
+ delta_cache[l] = ((qh & 8) != 0) ? -IQ1M_DELTA : IQ1M_DELTA;
42
+ }
43
+
44
+ // Loop over columns of the output
45
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
46
+ // Compute base index for matrix B
47
+ const uint base_b_idx = (j * p.batch_stride_b + b_offset + y_idx) / 4;
48
+ vec4 b_vals[8];
49
+
50
+ // Load 8 vec4 values from matrix B
51
+ [[unroll]] for (int idx = 0; idx < 8; ++idx) {
52
+ b_vals[idx] = vec4(data_b_v4[base_b_idx + idx]);
53
+ }
54
+
55
+ FLOAT_TYPE col_sum = FLOAT_TYPE(0.0);
56
+
57
+ // Loop over sub-blocks
58
+ [[unroll]] for (uint l = 0; l < 4; ++l) {
59
+ const uint16_t grid = gvf_cache[l];
60
+ const float dl = dl_cache[l];
61
+
62
+ // Decode 8 2-bit fbits from gvf_cache
63
+ float f0 = float(bitfieldExtract(grid, 0, 2));
64
+ float f1 = float(bitfieldExtract(grid, 2, 2));
65
+ float f2 = float(bitfieldExtract(grid, 4, 2));
66
+ float f3 = float(bitfieldExtract(grid, 6, 2));
67
+ float f4 = float(bitfieldExtract(grid, 8, 2));
68
+ float f5 = float(bitfieldExtract(grid, 10, 2));
69
+ float f6 = float(bitfieldExtract(grid, 12, 2));
70
+ float f7 = float(bitfieldExtract(grid, 14, 2));
71
+
72
+ // Pack into vec4 for vectorized FMA
73
+ const vec4 fbits_v0 = vec4(f0, f1, f2, f3);
74
+ const vec4 fbits_v1 = vec4(f4, f5, f6, f7);
75
+ const vec4 delta_v = vec4(delta_cache[l]);
76
+
77
+ // Vectorized fused multiply-add
78
+ vec4 sum_v = fma(b_vals[2*l + 0], fbits_v0 + delta_v, vec4(0.0));
79
+ sum_v = fma(b_vals[2*l + 1], fbits_v1 + delta_v, sum_v);
80
+
81
+ // Horizontal add to get scalar sum
82
+ FLOAT_TYPE sum = sum_v.x + sum_v.y + sum_v.z + sum_v.w;
83
+
84
+ // Accumulate to column sum
85
+ col_sum = fma(dl, sum, col_sum);
86
+ }
87
+ // Write result to temporary buffer
88
+ temp[j][n] += col_sum;
89
+ }
90
+ ibi += num_blocks_per_row;
91
+ }
92
+ }
93
+
94
+ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
95
+ uint a_offset, b_offset, d_offset;
96
+ get_offsets(a_offset, b_offset, d_offset);
97
+
98
+ const uint num_blocks_per_row = p.ncols / QUANT_K;
99
+
100
+ // 8 threads are used to process each block
101
+ const uint blocks_per_wg = gl_WorkGroupSize.x/8;
102
+ const uint tid = gl_LocalInvocationID.x;
103
+ const uint itid = tid % 8; // 0...7
104
+ const uint ix = tid / 8;
105
+
106
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
107
+ [[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) {
108
+ temp[j][i] = FLOAT_TYPE(0);
109
+ }
110
+ }
111
+
112
+ [[unroll]] for (uint i = ix; i < num_blocks_per_row; i += blocks_per_wg)
113
+ calc_superblock(a_offset, b_offset, itid, i, num_blocks_per_row, first_row, num_rows);
114
+
115
+ reduce_result(temp, d_offset, first_row, num_rows, tid);
116
+ }
117
+
118
+ void main() {
119
+ const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z);
120
+
121
+ init_iq_shmem(gl_WorkGroupSize);
122
+
123
+ // do NUM_ROWS at a time, unless there aren't enough remaining rows
124
+ if (first_row + NUM_ROWS <= p.stride_d) {
125
+ compute_outputs(first_row, NUM_ROWS);
126
+ } else {
127
+ if (first_row >= p.stride_d) {
128
+ return;
129
+ }
130
+ compute_outputs(first_row, p.stride_d - first_row);
131
+ }
132
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp ADDED
@@ -0,0 +1,95 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+ #extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
3
+
4
+ #include "mul_mat_vec_base.glsl"
5
+
6
+ layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
7
+
8
+ FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];
9
+
10
+ void calc_superblock(const uint a_offset, const uint b_offset, const uint ib32, const uint i,
11
+ const uint num_blocks_per_row, const uint first_row, const uint num_rows) {
12
+ const uint y_idx_base = i * QUANT_K + 32 * ib32;
13
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
14
+ const uint base_b_idx = (j * p.batch_stride_b + b_offset + y_idx_base) / 4;
15
+ [[unroll]] for (uint l = 0; l < 4; ++l) {
16
+ const vec4 b_val_0 = vec4(data_b_v4[base_b_idx + 2 * l]);
17
+ const vec4 b_val_1 = vec4(data_b_v4[base_b_idx + 2 * l + 1]);
18
+
19
+ // index for data_a
20
+ uint ibi = a_offset + first_row * num_blocks_per_row + i;
21
+
22
+ [[unroll]] for (uint n = 0; n < num_rows; ++n) {
23
+ const float d = float(data_a[ibi].d);
24
+ const uint qh = data_a[ibi].qh[ib32];
25
+
26
+ const float dl = d * float(2 * bitfieldExtract(qh, 12, 3) + 1);
27
+ const uint qs = data_a[ibi].qs[4 * ib32 + l];
28
+ const uint idxhi = bitfieldExtract(qh, 3 * int(l), 3);
29
+ const uint16_t grid = uint16_t(iq1s_grid[qs | (idxhi << 8)]);
30
+
31
+ const float delta_val = ((qh & 0x8000) != 0) ? -IQ1S_DELTA : IQ1S_DELTA;
32
+ const vec4 delta_v = vec4(delta_val);
33
+ const vec4 fbits0 = vec4(
34
+ float(bitfieldExtract(grid, 0, 2)),
35
+ float(bitfieldExtract(grid, 2, 2)),
36
+ float(bitfieldExtract(grid, 4, 2)),
37
+ float(bitfieldExtract(grid, 6, 2))
38
+ );
39
+ const vec4 fbits1 = vec4(
40
+ float(bitfieldExtract(grid, 8, 2)),
41
+ float(bitfieldExtract(grid, 10, 2)),
42
+ float(bitfieldExtract(grid, 12, 2)),
43
+ float(bitfieldExtract(grid, 14, 2))
44
+ );
45
+
46
+ vec4 sum_v = fma(b_val_0, fbits0 + delta_v, vec4(0.0));
47
+ sum_v = fma(b_val_1, fbits1 + delta_v, sum_v);
48
+ FLOAT_TYPE sum = dot(sum_v, vec4(1.0));
49
+
50
+ temp[j][n] = fma(dl, sum, temp[j][n]);
51
+ ibi += num_blocks_per_row;
52
+ }
53
+ }
54
+ }
55
+ }
56
+
57
+ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
58
+ uint a_offset, b_offset, d_offset;
59
+ get_offsets(a_offset, b_offset, d_offset);
60
+
61
+ const uint num_blocks_per_row = p.ncols / QUANT_K;
62
+
63
+ // 8 threads are used to process each block
64
+ const uint blocks_per_wg = gl_WorkGroupSize.x/8;
65
+ const uint tid = gl_LocalInvocationID.x;
66
+ const uint itid = tid % 8; // 0...7
67
+ const uint ix = tid / 8;
68
+
69
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
70
+ [[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) {
71
+ temp[j][i] = FLOAT_TYPE(0);
72
+ }
73
+ }
74
+
75
+ [[unroll]] for (uint i = ix; i < num_blocks_per_row; i += blocks_per_wg)
76
+ calc_superblock(a_offset, b_offset, itid, i, num_blocks_per_row, first_row, num_rows);
77
+
78
+ reduce_result(temp, d_offset, first_row, num_rows, tid);
79
+ }
80
+
81
+ void main() {
82
+ const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z);
83
+
84
+ init_iq_shmem(gl_WorkGroupSize);
85
+
86
+ // do NUM_ROWS at a time, unless there aren't enough remaining rows
87
+ if (first_row + NUM_ROWS <= p.stride_d) {
88
+ compute_outputs(first_row, NUM_ROWS);
89
+ } else {
90
+ if (first_row >= p.stride_d) {
91
+ return;
92
+ }
93
+ compute_outputs(first_row, p.stride_d - first_row);
94
+ }
95
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp ADDED
@@ -0,0 +1,90 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+ #extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
3
+
4
+ #include "mul_mat_vec_base.glsl"
5
+
6
+ layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
7
+
8
+ FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];
9
+
10
+ void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) {
11
+ const uint y_idx = i * QUANT_K + 16 * itid;
12
+ const uint nibble_shift = 4 * (itid & 1);
13
+ const uint ib32 = itid / 2; // 0..7
14
+
15
+ uint ibi = a_offset + first_row * num_blocks_per_row + i;
16
+ [[unroll]] for (uint n = 0; n < num_rows; ++n) {
17
+ const float d = float(data_a[ibi].d);
18
+ const uint scale = (data_a[ibi].scales[ib32] >> nibble_shift) & 0xF;
19
+ const float db = d * (0.5 + scale) * 0.25;
20
+
21
+ const uint qh = data_a[ibi].qh[ib32];
22
+ const u8vec2 qs16 = unpack8(uint32_t(data_a_packed16[ibi].qs[itid])).xy; // vec4 used due to #12147
23
+ const u8vec2 sign16 = unpack8(uint32_t(data_a_packed16[ibi].qs[QUANT_K / 16 + itid])).xy;
24
+ [[unroll]] for (uint l = 0; l < 2; ++l) {
25
+ const uint8_t sign = sign16[l];
26
+ const uint qs = qs16[l] | ((qh << (8 - nibble_shift - 2 * l)) & 0x300);
27
+ const uvec2 grid = iq2s_grid[qs];
28
+ const vec4 grid0 = vec4(unpack8(grid.x));
29
+ const vec4 grid1 = vec4(unpack8(grid.y));
30
+
31
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
32
+ vec4 b0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 0]);
33
+ vec4 b4 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 1]);
34
+
35
+ FLOAT_TYPE sum =
36
+ fma(FLOAT_TYPE(b0.x), FLOAT_TYPE((sign & 1) != 0 ? -grid0.x : grid0.x),
37
+ fma(FLOAT_TYPE(b0.y), FLOAT_TYPE((sign & 2) != 0 ? -grid0.y : grid0.y),
38
+ fma(FLOAT_TYPE(b0.z), FLOAT_TYPE((sign & 4) != 0 ? -grid0.z : grid0.z),
39
+ fma(FLOAT_TYPE(b0.w), FLOAT_TYPE((sign & 8) != 0 ? -grid0.w : grid0.w),
40
+ fma(FLOAT_TYPE(b4.x), FLOAT_TYPE((sign & 16) != 0 ? -grid1.x : grid1.x),
41
+ fma(FLOAT_TYPE(b4.y), FLOAT_TYPE((sign & 32) != 0 ? -grid1.y : grid1.y),
42
+ fma(FLOAT_TYPE(b4.z), FLOAT_TYPE((sign & 64) != 0 ? -grid1.z : grid1.z),
43
+ fma(FLOAT_TYPE(b4.w), FLOAT_TYPE((sign & 128) != 0 ? -grid1.w : grid1.w),
44
+ FLOAT_TYPE(0.0)))))))));
45
+ temp[j][n] = fma(db, sum, temp[j][n]);
46
+ }
47
+ }
48
+ ibi += num_blocks_per_row;
49
+ }
50
+ }
51
+
52
+ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
53
+ uint a_offset, b_offset, d_offset;
54
+ get_offsets(a_offset, b_offset, d_offset);
55
+
56
+ const uint num_blocks_per_row = p.ncols / QUANT_K;
57
+
58
+ // 16 threads are used to process each block
59
+ const uint blocks_per_wg = gl_WorkGroupSize.x/16;
60
+ const uint tid = gl_LocalInvocationID.x;
61
+ const uint itid = tid % 16; // 0...15
62
+ const uint ix = tid / 16;
63
+
64
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
65
+ [[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) {
66
+ temp[j][i] = FLOAT_TYPE(0);
67
+ }
68
+ }
69
+
70
+ [[unroll]] for (uint i = ix; i < num_blocks_per_row; i += blocks_per_wg)
71
+ calc_superblock(a_offset, b_offset, itid, i, num_blocks_per_row, first_row, num_rows);
72
+
73
+ reduce_result(temp, d_offset, first_row, num_rows, tid);
74
+ }
75
+
76
+ void main() {
77
+ const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z);
78
+
79
+ init_iq_shmem(gl_WorkGroupSize);
80
+
81
+ // do NUM_ROWS at a time, unless there aren't enough remaining rows
82
+ if (first_row + NUM_ROWS <= p.stride_d) {
83
+ compute_outputs(first_row, NUM_ROWS);
84
+ } else {
85
+ if (first_row >= p.stride_d) {
86
+ return;
87
+ }
88
+ compute_outputs(first_row, p.stride_d - first_row);
89
+ }
90
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp ADDED
@@ -0,0 +1,105 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+ #extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
3
+
4
+ #include "mul_mat_vec_base.glsl"
5
+
6
+ layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
7
+
8
+ FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];
9
+
10
+ void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) {
11
+ const uint y_idx = i * QUANT_K + 16 * itid;
12
+ const uint nibble_shift = 4 * (itid & 1);
13
+ const uint ib32 = itid / 2; // 0..7
14
+ uint ibi = a_offset + first_row * num_blocks_per_row + i;
15
+ // Precompute db multiplication factors
16
+ float db_vals[NUM_ROWS];
17
+ [[unroll]] for (uint n = 0; n < num_rows; ++n) {
18
+ const float d = float(data_a[ibi].d);
19
+ const uint scale_raw = data_a[ibi].scales[ib32];
20
+ const uint scale = (scale_raw >> nibble_shift) & 0xF;
21
+ // Merge constant calculations d * (0.5 + scale) * 0.25 = d*0.125 + d*scale*0.25
22
+ db_vals[n] = d * (0.125f + float(scale) * 0.25f);
23
+ ibi += num_blocks_per_row;
24
+ }
25
+ ibi = a_offset + first_row * num_blocks_per_row + i;
26
+ [[unroll]] for (uint n = 0; n < num_rows; ++n) {
27
+ // Preload grid and sign data for all l values
28
+ vec4 grid0_vals[2], grid1_vals[2];
29
+ uint sign_vals[2], sign7_vals[2];
30
+ [[unroll]] for (uint l = 0; l < 2; ++l) {
31
+ const uint qs = data_a[ibi].qs[2 * itid + l];
32
+ sign_vals[l] = qs >> 9;
33
+ sign7_vals[l] = bitCount(sign_vals[l]);
34
+ const uvec2 grid_data = iq2xs_grid[qs & 511];
35
+ grid0_vals[l] = vec4(unpack8(grid_data.x));
36
+ grid1_vals[l] = vec4(unpack8(grid_data.y));
37
+ }
38
+ // Preload B data for all j columns (reduce repeated index calculations)
39
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
40
+ FLOAT_TYPE sum = FLOAT_TYPE(0.0);
41
+ [[unroll]] for (uint l = 0; l < 2; ++l) {
42
+ const uint sign = sign_vals[l];
43
+ const uint sign7 = sign7_vals[l];
44
+ const vec4 grid0 = grid0_vals[l];
45
+ const vec4 grid1 = grid1_vals[l];
46
+ // Precompute indices
47
+ const uint b_idx = (j * p.batch_stride_b + b_offset + y_idx) / 4 + 2 * l;
48
+ const vec4 b0 = vec4(data_b_v4[b_idx + 0]);
49
+ const vec4 b4 = vec4(data_b_v4[b_idx + 1]);
50
+ sum +=
51
+ fma(FLOAT_TYPE(b0.x), FLOAT_TYPE((sign & 1) != 0 ? -grid0.x : grid0.x),
52
+ fma(FLOAT_TYPE(b0.y), FLOAT_TYPE((sign & 2) != 0 ? -grid0.y : grid0.y),
53
+ fma(FLOAT_TYPE(b0.z), FLOAT_TYPE((sign & 4) != 0 ? -grid0.z : grid0.z),
54
+ fma(FLOAT_TYPE(b0.w), FLOAT_TYPE((sign & 8) != 0 ? -grid0.w : grid0.w),
55
+ fma(FLOAT_TYPE(b4.x), FLOAT_TYPE((sign & 16) != 0 ? -grid1.x : grid1.x),
56
+ fma(FLOAT_TYPE(b4.y), FLOAT_TYPE((sign & 32) != 0 ? -grid1.y : grid1.y),
57
+ fma(FLOAT_TYPE(b4.z), FLOAT_TYPE((sign & 64) != 0 ? -grid1.z : grid1.z),
58
+ fma(FLOAT_TYPE(b4.w), FLOAT_TYPE((sign7 & 1) != 0 ? -grid1.w : grid1.w),
59
+ FLOAT_TYPE(0.0)))))))));
60
+ }
61
+ temp[j][n] = fma(FLOAT_TYPE(db_vals[n]), sum, temp[j][n]);
62
+ }
63
+ ibi += num_blocks_per_row;
64
+ }
65
+ }
66
+
67
+ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
68
+ uint a_offset, b_offset, d_offset;
69
+ get_offsets(a_offset, b_offset, d_offset);
70
+
71
+ const uint num_blocks_per_row = p.ncols / QUANT_K;
72
+
73
+ // 16 threads are used to process each block
74
+ const uint blocks_per_wg = gl_WorkGroupSize.x/16;
75
+ const uint tid = gl_LocalInvocationID.x;
76
+ const uint itid = tid % 16; // 0...15
77
+ const uint ix = tid / 16;
78
+
79
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
80
+ [[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) {
81
+ temp[j][i] = FLOAT_TYPE(0);
82
+ }
83
+ }
84
+
85
+ [[unroll]] for (uint i = ix; i < num_blocks_per_row; i += blocks_per_wg)
86
+ calc_superblock(a_offset, b_offset, itid, i, num_blocks_per_row, first_row, num_rows);
87
+
88
+ reduce_result(temp, d_offset, first_row, num_rows, tid);
89
+ }
90
+
91
+ void main() {
92
+ const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z);
93
+
94
+ init_iq_shmem(gl_WorkGroupSize);
95
+
96
+ // do NUM_ROWS at a time, unless there aren't enough remaining rows
97
+ if (first_row + NUM_ROWS <= p.stride_d) {
98
+ compute_outputs(first_row, NUM_ROWS);
99
+ } else {
100
+ if (first_row >= p.stride_d) {
101
+ return;
102
+ }
103
+ compute_outputs(first_row, p.stride_d - first_row);
104
+ }
105
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp ADDED
@@ -0,0 +1,87 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+ #extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
3
+
4
+ #include "mul_mat_vec_base.glsl"
5
+
6
+ layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
7
+
8
+ FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];
9
+
10
+ void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) {
11
+ const uint y_idx = i * QUANT_K + 16 * itid;
12
+ const uint ib32 = itid / 2; // 0..7
13
+
14
+ uint ibi = a_offset + first_row * num_blocks_per_row + i;
15
+ [[unroll]] for (uint n = 0; n < num_rows; ++n) {
16
+ const float d = float(data_a[ibi].d);
17
+ const uint signscale = pack32(u16vec2(
18
+ data_a_packed16[ibi].qs[4 * ib32 + 2],
19
+ data_a_packed16[ibi].qs[4 * ib32 + 3]));
20
+ const float db = d * 0.25 * (0.5 + (signscale >> 28));
21
+ [[unroll]] for (uint l = 0; l < 2; ++l) {
22
+ const uint qs = data_a[ibi].qs[8 * ib32 + 2 * (itid & 1) + l];
23
+ const uint sign = bitfieldExtract(signscale, 7 * int(2 * (itid & 1) + l), 7);
24
+ const uint sign7 = bitCount(sign);
25
+ const vec4 grid0 = vec4(unpack8(iq2xxs_grid[qs].x));
26
+ const vec4 grid1 = vec4(unpack8(iq2xxs_grid[qs].y));
27
+
28
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
29
+ const vec4 b0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 0]);
30
+ const vec4 b4 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 1]);
31
+
32
+ FLOAT_TYPE sum =
33
+ fma(FLOAT_TYPE(b0.x), FLOAT_TYPE((sign & 1) != 0 ? -grid0.x : grid0.x),
34
+ fma(FLOAT_TYPE(b0.y), FLOAT_TYPE((sign & 2) != 0 ? -grid0.y : grid0.y),
35
+ fma(FLOAT_TYPE(b0.z), FLOAT_TYPE((sign & 4) != 0 ? -grid0.z : grid0.z),
36
+ fma(FLOAT_TYPE(b0.w), FLOAT_TYPE((sign & 8) != 0 ? -grid0.w : grid0.w),
37
+ fma(FLOAT_TYPE(b4.x), FLOAT_TYPE((sign & 16) != 0 ? -grid1.x : grid1.x),
38
+ fma(FLOAT_TYPE(b4.y), FLOAT_TYPE((sign & 32) != 0 ? -grid1.y : grid1.y),
39
+ fma(FLOAT_TYPE(b4.z), FLOAT_TYPE((sign & 64) != 0 ? -grid1.z : grid1.z),
40
+ fma(FLOAT_TYPE(b4.w), FLOAT_TYPE((sign7 & 1) != 0 ? -grid1.w : grid1.w),
41
+ FLOAT_TYPE(0.0)))))))));
42
+ temp[j][n] = fma(db, sum, temp[j][n]);
43
+ }
44
+ }
45
+ ibi += num_blocks_per_row;
46
+ }
47
+ }
48
+
49
+ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
50
+ uint a_offset, b_offset, d_offset;
51
+ get_offsets(a_offset, b_offset, d_offset);
52
+
53
+ const uint num_blocks_per_row = p.ncols / QUANT_K;
54
+
55
+ // 16 threads are used to process each block
56
+ const uint blocks_per_wg = gl_WorkGroupSize.x/16;
57
+ const uint tid = gl_LocalInvocationID.x;
58
+ const uint itid = tid % 16; // 0...15
59
+ const uint ix = tid / 16;
60
+
61
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
62
+ [[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) {
63
+ temp[j][i] = FLOAT_TYPE(0);
64
+ }
65
+ }
66
+
67
+ [[unroll]] for (uint i = ix; i < num_blocks_per_row; i += blocks_per_wg)
68
+ calc_superblock(a_offset, b_offset, itid, i, num_blocks_per_row, first_row, num_rows);
69
+
70
+ reduce_result(temp, d_offset, first_row, num_rows, tid);
71
+ }
72
+
73
+ void main() {
74
+ const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z);
75
+
76
+ init_iq_shmem(gl_WorkGroupSize);
77
+
78
+ // do NUM_ROWS at a time, unless there aren't enough remaining rows
79
+ if (first_row + NUM_ROWS <= p.stride_d) {
80
+ compute_outputs(first_row, NUM_ROWS);
81
+ } else {
82
+ if (first_row >= p.stride_d) {
83
+ return;
84
+ }
85
+ compute_outputs(first_row, p.stride_d - first_row);
86
+ }
87
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp ADDED
@@ -0,0 +1,90 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+ #extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
3
+
4
+ #include "mul_mat_vec_base.glsl"
5
+
6
+ layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
7
+
8
+ FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];
9
+
10
+ void calc_superblock(const uint a_offset, const uint b_offset, const uint ib32, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) {
11
+ const uint y_idx = i * QUANT_K + 32 * ib32;
12
+
13
+ uint ibi = a_offset + first_row * num_blocks_per_row + i;
14
+ [[unroll]] for (uint n = 0; n < num_rows; ++n) {
15
+ const float d = float(data_a[ibi].d);
16
+ const uint scale = (data_a[ibi].scales[ib32/2] >> (4 * (ib32 & 1))) & 0xF;
17
+ const float dscale = d * (1 + 2 * scale);
18
+ const uint qh = data_a[ibi].qh[ib32];
19
+ FLOAT_TYPE sum[NUM_COLS];
20
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
21
+ sum[j] = 0.0;
22
+ }
23
+ [[unroll]] for (uint l = 0; l < 4; ++l) {
24
+ const u8vec2 qs = unpack8(uint32_t(data_a_packed16[ibi].qs[4 * ib32 + l])).xy; // vec4 used due to #12147
25
+ const uint sign = data_a[ibi].signs[4 * ib32 + l];
26
+ const vec4 grid0 = vec4(unpack8(iq3s_grid[qs.x | ((qh << (8 - 2*l)) & 0x100)]));
27
+ const vec4 grid1 = vec4(unpack8(iq3s_grid[qs.y | ((qh << (7 - 2*l)) & 0x100)]));
28
+
29
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
30
+ const vec4 b0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 0]);
31
+ const vec4 b4 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 1]);
32
+
33
+ sum[j] =
34
+ fma(FLOAT_TYPE(b0.x), FLOAT_TYPE((sign & 1) != 0 ? -grid0.x : grid0.x),
35
+ fma(FLOAT_TYPE(b0.y), FLOAT_TYPE((sign & 2) != 0 ? -grid0.y : grid0.y),
36
+ fma(FLOAT_TYPE(b0.z), FLOAT_TYPE((sign & 4) != 0 ? -grid0.z : grid0.z),
37
+ fma(FLOAT_TYPE(b0.w), FLOAT_TYPE((sign & 8) != 0 ? -grid0.w : grid0.w),
38
+ fma(FLOAT_TYPE(b4.x), FLOAT_TYPE((sign & 16) != 0 ? -grid1.x : grid1.x),
39
+ fma(FLOAT_TYPE(b4.y), FLOAT_TYPE((sign & 32) != 0 ? -grid1.y : grid1.y),
40
+ fma(FLOAT_TYPE(b4.z), FLOAT_TYPE((sign & 64) != 0 ? -grid1.z : grid1.z),
41
+ fma(FLOAT_TYPE(b4.w), FLOAT_TYPE((sign & 128) != 0 ? -grid1.w : grid1.w),
42
+ sum[j]))))))));
43
+ }
44
+ }
45
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
46
+ temp[j][n] = fma(dscale, sum[j], temp[j][n]);
47
+ }
48
+ ibi += num_blocks_per_row;
49
+ }
50
+ }
51
+
52
+ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
53
+ uint a_offset, b_offset, d_offset;
54
+ get_offsets(a_offset, b_offset, d_offset);
55
+
56
+ const uint num_blocks_per_row = p.ncols / QUANT_K;
57
+
58
+ // 8 threads are used to process each block
59
+ const uint blocks_per_wg = gl_WorkGroupSize.x/8;
60
+ const uint tid = gl_LocalInvocationID.x;
61
+ const uint itid = tid % 8; // 0...7
62
+ const uint ix = tid / 8;
63
+
64
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
65
+ [[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) {
66
+ temp[j][i] = FLOAT_TYPE(0);
67
+ }
68
+ }
69
+
70
+ [[unroll]] for (uint i = ix; i < num_blocks_per_row; i += blocks_per_wg)
71
+ calc_superblock(a_offset, b_offset, itid, i, num_blocks_per_row, first_row, num_rows);
72
+
73
+ reduce_result(temp, d_offset, first_row, num_rows, tid);
74
+ }
75
+
76
+ void main() {
77
+ const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z);
78
+
79
+ init_iq_shmem(gl_WorkGroupSize);
80
+
81
+ // do NUM_ROWS at a time, unless there aren't enough remaining rows
82
+ if (first_row + NUM_ROWS <= p.stride_d) {
83
+ compute_outputs(first_row, NUM_ROWS);
84
+ } else {
85
+ if (first_row >= p.stride_d) {
86
+ return;
87
+ }
88
+ compute_outputs(first_row, p.stride_d - first_row);
89
+ }
90
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp ADDED
@@ -0,0 +1,88 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+ #extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
3
+
4
+ #include "mul_mat_vec_base.glsl"
5
+
6
+ layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
7
+
8
+ FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];
9
+
10
+ void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) {
11
+ const uint y_idx = i * QUANT_K + 16 * itid;
12
+ const uint ib32 = itid / 2; // 0..7
13
+
14
+ uint ibi = a_offset + first_row * num_blocks_per_row + i;
15
+ [[unroll]] for (uint n = 0; n < num_rows; ++n) {
16
+ const float d = float(data_a[ibi].d);
17
+ const uint signscale = pack32(u16vec2(
18
+ data_a_packed16[ibi].qs[QUANT_K / 8 + 2 * ib32],
19
+ data_a_packed16[ibi].qs[QUANT_K / 8 + 2 * ib32 + 1]));
20
+ const float db = d * 0.5 * (0.5 + (signscale >> 28));
21
+ [[unroll]] for (uint l = 0; l < 2; ++l) {
22
+ const uint qs0 = data_a[ibi].qs[8 * ib32 + 4 * (itid & 1) + 2 * l];
23
+ const uint qs1 = data_a[ibi].qs[8 * ib32 + 4 * (itid & 1) + 2 * l + 1];
24
+ const uint sign = bitfieldExtract(signscale, 7 * int(2 * (itid & 1) + l), 7);
25
+ const uint sign7 = bitCount(sign);
26
+ const vec4 grid0 = vec4(unpack8(iq3xxs_grid[qs0]));
27
+ const vec4 grid1 = vec4(unpack8(iq3xxs_grid[qs1]));
28
+
29
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
30
+ const vec4 b0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 0]);
31
+ const vec4 b4 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 1]);
32
+
33
+ FLOAT_TYPE sum =
34
+ fma(FLOAT_TYPE(b0.x), FLOAT_TYPE((sign & 1) != 0 ? -grid0.x : grid0.x),
35
+ fma(FLOAT_TYPE(b0.y), FLOAT_TYPE((sign & 2) != 0 ? -grid0.y : grid0.y),
36
+ fma(FLOAT_TYPE(b0.z), FLOAT_TYPE((sign & 4) != 0 ? -grid0.z : grid0.z),
37
+ fma(FLOAT_TYPE(b0.w), FLOAT_TYPE((sign & 8) != 0 ? -grid0.w : grid0.w),
38
+ fma(FLOAT_TYPE(b4.x), FLOAT_TYPE((sign & 16) != 0 ? -grid1.x : grid1.x),
39
+ fma(FLOAT_TYPE(b4.y), FLOAT_TYPE((sign & 32) != 0 ? -grid1.y : grid1.y),
40
+ fma(FLOAT_TYPE(b4.z), FLOAT_TYPE((sign & 64) != 0 ? -grid1.z : grid1.z),
41
+ fma(FLOAT_TYPE(b4.w), FLOAT_TYPE((sign7 & 1) != 0 ? -grid1.w : grid1.w),
42
+ FLOAT_TYPE(0.0)))))))));
43
+ temp[j][n] = fma(db, sum, temp[j][n]);
44
+ }
45
+ }
46
+ ibi += num_blocks_per_row;
47
+ }
48
+ }
49
+
50
+ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
51
+ uint a_offset, b_offset, d_offset;
52
+ get_offsets(a_offset, b_offset, d_offset);
53
+
54
+ const uint num_blocks_per_row = p.ncols / QUANT_K;
55
+
56
+ // 16 threads are used to process each block
57
+ const uint blocks_per_wg = gl_WorkGroupSize.x/16;
58
+ const uint tid = gl_LocalInvocationID.x;
59
+ const uint itid = tid % 16; // 0...15
60
+ const uint ix = tid / 16;
61
+
62
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
63
+ [[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) {
64
+ temp[j][i] = FLOAT_TYPE(0);
65
+ }
66
+ }
67
+
68
+ [[unroll]] for (uint i = ix; i < num_blocks_per_row; i += blocks_per_wg)
69
+ calc_superblock(a_offset, b_offset, itid, i, num_blocks_per_row, first_row, num_rows);
70
+
71
+ reduce_result(temp, d_offset, first_row, num_rows, tid);
72
+ }
73
+
74
+ void main() {
75
+ const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z);
76
+
77
+ init_iq_shmem(gl_WorkGroupSize);
78
+
79
+ // do NUM_ROWS at a time, unless there aren't enough remaining rows
80
+ if (first_row + NUM_ROWS <= p.stride_d) {
81
+ compute_outputs(first_row, NUM_ROWS);
82
+ } else {
83
+ if (first_row >= p.stride_d) {
84
+ return;
85
+ }
86
+ compute_outputs(first_row, p.stride_d - first_row);
87
+ }
88
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp ADDED
@@ -0,0 +1,124 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+
3
+ #extension GL_EXT_control_flow_attributes : enable
4
+ #extension GL_EXT_shader_16bit_storage : require
5
+
6
+ #define BLOCK_SIZE 32
7
+ #define FLOAT_TYPE float
8
+
9
+ layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in;
10
+
11
+ #include "mul_mat_vec_iface.glsl"
12
+
13
+ layout (push_constant) uniform parameter
14
+ {
15
+ uint ncols_x;
16
+ uint nrows_x;
17
+ uint row_stride_x;
18
+ uint channel_stride_x;
19
+ uint channel_stride_y;
20
+ uint channel_x_divisor;
21
+ uint ne12;
22
+ uint b_offset;
23
+ uint d_offset;
24
+ uint nb03;
25
+ uint nb13;
26
+ uint nb23;
27
+ uint fusion_flags;
28
+ } p;
29
+
30
+ shared FLOAT_TYPE tmp[BLOCK_SIZE];
31
+
32
+ void main() {
33
+ const uint tid = gl_LocalInvocationID.x;
34
+ const uint row_x = gl_GlobalInvocationID.y;
35
+ const uint channel = gl_GlobalInvocationID.z;
36
+ const uint i3 = gl_WorkGroupID.x;
37
+ const uint channel_x = channel / p.channel_x_divisor;
38
+ const uint channel_y = channel % p.ne12;
39
+
40
+ const uint nrows_y = p.ncols_x;
41
+ const uint nrows_dst = p.nrows_x;
42
+ const uint row_dst = row_x;
43
+
44
+ const uint idst = i3*p.nb23 + channel*nrows_dst + row_dst;
45
+
46
+ FLOAT_TYPE temp = 0.0f;
47
+
48
+ // Detect alignment for vector loads
49
+ bool is_aligned = (p.ncols_x % 4) == 0 && (p.row_stride_x % 4) == 0 && (p.channel_stride_x % 4) == 0;
50
+
51
+ for (uint col_x0 = 0; col_x0 < p.ncols_x;) {
52
+
53
+ // Unroll 2x and do vec4 loads if aligned
54
+ const uint unroll_count = 2;
55
+ if (col_x0 + unroll_count * 4 * BLOCK_SIZE <= p.ncols_x && is_aligned) {
56
+ [[unroll]] for (uint i = 0; i < unroll_count; ++i) {
57
+ const uint col_x = col_x0 + 4*tid;
58
+
59
+ const uint row_y = col_x;
60
+
61
+ const uint ix = i3*p.nb03 + channel_x*p.channel_stride_x + row_x*p.row_stride_x + col_x;
62
+ const uint iy = i3*p.nb13 + channel_y*p.channel_stride_y + row_y;
63
+
64
+ const vec4 av4 = vec4(data_a_v4[ix / 4]);
65
+ const vec4 bv4 = vec4(data_b_v4[iy / 4]);
66
+
67
+ temp += dot(av4, bv4);
68
+
69
+ col_x0 += 4*BLOCK_SIZE;
70
+ }
71
+ // do vec4 loads if aligned
72
+ } else if (col_x0 + 4*BLOCK_SIZE <= p.ncols_x && is_aligned) {
73
+ const uint col_x = col_x0 + 4*tid;
74
+
75
+ const uint row_y = col_x;
76
+
77
+ const uint ix = i3*p.nb03 + channel_x*p.channel_stride_x + row_x*p.row_stride_x + col_x;
78
+ const uint iy = i3*p.nb13 + channel_y*p.channel_stride_y + row_y;
79
+
80
+ const vec4 av4 = vec4(data_a_v4[ix / 4]);
81
+ const vec4 bv4 = vec4(data_b_v4[iy / 4]);
82
+
83
+ temp += dot(av4, bv4);
84
+
85
+ col_x0 += 4*BLOCK_SIZE;
86
+ } else {
87
+ const uint col_x = col_x0 + tid;
88
+ if (col_x >= p.ncols_x) {
89
+ break;
90
+ }
91
+
92
+ const uint row_y = col_x;
93
+
94
+ const uint ix = i3*p.nb03 + channel_x*p.channel_stride_x + row_x*p.row_stride_x + col_x;
95
+ const uint iy = i3*p.nb13 + channel_y*p.channel_stride_y + row_y;
96
+
97
+ const FLOAT_TYPE xi = FLOAT_TYPE(data_a[ix]);
98
+
99
+ temp = fma(xi, FLOAT_TYPE(data_b[iy]), temp);
100
+ col_x0 += BLOCK_SIZE;
101
+ }
102
+ }
103
+
104
+ tmp[tid] = temp;
105
+
106
+ // sum up partial sums and write back result
107
+ barrier();
108
+ [[unroll]] for (int s = BLOCK_SIZE / 2; s > 0; s >>= 1) {
109
+ if (tid < s) {
110
+ tmp[tid] += tmp[tid + s];
111
+ }
112
+ barrier();
113
+ }
114
+
115
+ if (tid == 0) {
116
+ if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0) {
117
+ tmp[0] += FLOAT_TYPE(data_fuse0[idst]);
118
+ }
119
+ if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS1) != 0) {
120
+ tmp[0] += FLOAT_TYPE(data_fuse1[idst]);
121
+ }
122
+ data_d[idst] = tmp[0];
123
+ }
124
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp ADDED
@@ -0,0 +1,156 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+
3
+ #extension GL_EXT_control_flow_attributes : enable
4
+ #extension GL_EXT_shader_16bit_storage : require
5
+ #if USE_SUBGROUP_ADD
6
+ #extension GL_KHR_shader_subgroup_arithmetic : enable
7
+ #endif
8
+
9
+ #define FLOAT_TYPE float
10
+
11
+ layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
12
+
13
+ #include "mul_mat_vec_iface.glsl"
14
+
15
+ layout(constant_id = 0) const int BLOCK_SIZE = 32;
16
+ // gqa_ratio is in the range [1,8]
17
+ layout(constant_id = 1) const uint gqa_ratio = 1;
18
+
19
+ layout (push_constant) uniform parameter
20
+ {
21
+ uint ncols_x;
22
+ uint nrows_x;
23
+ uint nchannels_x;
24
+ uint nchannels_y;
25
+ uint b_offset;
26
+ uint d_offset;
27
+ uint fusion_flags;
28
+ } p;
29
+
30
+ #if !USE_SUBGROUP_ADD
31
+ shared FLOAT_TYPE tmp[8][BLOCK_SIZE];
32
+ #endif
33
+
34
+ void main() {
35
+ const uint tid = gl_LocalInvocationID.x;
36
+ const uint row_x = gl_GlobalInvocationID.y;
37
+
38
+ uint channel, channel_x;
39
+
40
+ // When gqa_ratio > 1, each invocation does multiple rows.
41
+ // The row in the A matrix is starting from channel / gqa_ratio and the
42
+ // rows in the B matrix are [channel, channel+gqa_ratio).
43
+ // When gpa_ratio is 1, each invocation does one row.
44
+ if (gqa_ratio > 1) {
45
+ channel_x = gl_GlobalInvocationID.z;
46
+ channel = channel_x * gqa_ratio;
47
+ } else {
48
+ channel = gl_GlobalInvocationID.z;
49
+ channel_x = channel / (p.nchannels_y / p.nchannels_x);;
50
+ }
51
+
52
+ const uint nrows_y = p.ncols_x;
53
+ const uint nrows_dst = p.nrows_x;
54
+ const uint row_dst = row_x;
55
+
56
+ FLOAT_TYPE temp[8];
57
+ [[unroll]] for (uint i = 0; i < 8; ++i) {
58
+ temp[i] = FLOAT_TYPE(0.0f);
59
+ }
60
+
61
+ // Detect alignment for vector loads
62
+ bool is_aligned = (p.ncols_x % 4) == 0 && (p.nchannels_x % 4) == 0 && (nrows_y % 4) == 0;
63
+
64
+ for (uint col_x0 = 0; col_x0 < p.ncols_x; col_x0 += BLOCK_SIZE) {
65
+
66
+ // Use vec4 loads if aligned
67
+ if (col_x0 + 4*BLOCK_SIZE <= p.ncols_x && is_aligned) {
68
+
69
+ uint col_x = col_x0 + 4*tid;
70
+ const uint row_y = col_x;
71
+
72
+ // x is transposed and permuted
73
+ const uint ix = row_x*p.nchannels_x*p.ncols_x + channel_x*p.ncols_x + col_x;
74
+ const vec4 av4 = vec4(data_a_v4[ix / 4]);
75
+
76
+ [[unroll]] for (uint c = 0; c < gqa_ratio; ++c) {
77
+ // y is not transposed but permuted
78
+ const uint iy = (channel + c)*nrows_y + row_y;
79
+
80
+ vec4 bv4 = data_b_v4[iy / 4];
81
+ temp[c] += dot(av4, bv4);
82
+ }
83
+
84
+ col_x0 += 3*BLOCK_SIZE;
85
+ } else {
86
+ const uint col_x = col_x0 + tid;
87
+
88
+ if (col_x >= p.ncols_x) {
89
+ break;
90
+ }
91
+
92
+ // x is transposed and permuted
93
+ const uint ix = row_x*p.nchannels_x*p.ncols_x + channel_x*p.ncols_x + col_x;
94
+ const FLOAT_TYPE xi = FLOAT_TYPE(data_a[ix]);
95
+
96
+ const uint row_y = col_x;
97
+
98
+ [[unroll]] for (uint c = 0; c < gqa_ratio; ++c) {
99
+ // y is not transposed but permuted
100
+ const uint iy = (channel + c)*nrows_y + row_y;
101
+
102
+ temp[c] = fma(xi, FLOAT_TYPE(data_b[iy]), temp[c]);
103
+ }
104
+ }
105
+ }
106
+
107
+ #if USE_SUBGROUP_ADD
108
+ // reduce vec4 at a time
109
+ vec4 t = vec4(temp[0], temp[1], temp[2], temp[3]);
110
+ t = subgroupAdd(t);
111
+ temp[0] = t[0];
112
+ temp[1] = t[1];
113
+ temp[2] = t[2];
114
+ temp[3] = t[3];
115
+ if (gqa_ratio > 4) {
116
+ t = vec4(temp[4], temp[5], temp[6], temp[7]);
117
+ t = subgroupAdd(t);
118
+ temp[4] = t[0];
119
+ temp[5] = t[1];
120
+ temp[6] = t[2];
121
+ temp[7] = t[3];
122
+ }
123
+ #else
124
+ [[unroll]] for (uint c = 0; c < gqa_ratio; ++c) {
125
+ tmp[c][tid] = temp[c];
126
+ }
127
+ // sum up partial sums and write back result
128
+ barrier();
129
+ [[unroll]] for (int s = BLOCK_SIZE / 2; s > 0; s >>= 1) {
130
+ if (tid < s) {
131
+ [[unroll]] for (uint c = 0; c < gqa_ratio; ++c) {
132
+ temp[c] += tmp[c][tid + s];
133
+ tmp[c][tid] = temp[c];
134
+ }
135
+ }
136
+ barrier();
137
+ }
138
+ [[unroll]] for (uint c = 0; c < gqa_ratio; ++c) {
139
+ temp[c] = tmp[c][tid];
140
+ }
141
+ #endif
142
+
143
+ if (tid == 0) {
144
+ [[unroll]] for (uint c = 0; c < gqa_ratio; ++c) {
145
+ // dst is not transposed and not permuted
146
+ const uint idst = (channel + c)*nrows_dst + row_dst;
147
+ if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0) {
148
+ temp[c] += FLOAT_TYPE(data_fuse0[idst]);
149
+ }
150
+ if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS1) != 0) {
151
+ temp[c] += FLOAT_TYPE(data_fuse1[idst]);
152
+ }
153
+ data_d[idst] = temp[c];
154
+ }
155
+ }
156
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp ADDED
@@ -0,0 +1,128 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+ #extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
3
+
4
+ #include "mul_mat_vec_base.glsl"
5
+
6
+ layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
7
+
8
+ shared FLOAT_TYPE sccache1[2][BLOCK_SIZE/16][16];
9
+ shared FLOAT_TYPE sccache2[2][BLOCK_SIZE/16][16];
10
+
11
+ FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];
12
+ uint csel = 0;
13
+
14
+ void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint v_im, const uint ix, const uint q_offset, const uint y_offset, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows, const bool all_threads) {
15
+ const uint y_idx = i * QUANT_K + y_offset;
16
+
17
+ [[unroll]] for (uint n = 0; n < num_rows; ++n) {
18
+ const uint ib0 = a_offset + (first_row+n)*num_blocks_per_row;
19
+ csel ^= 1;
20
+
21
+ if (!all_threads) { // when we don't have enough blocks to use all threads
22
+ if (i < num_blocks_per_row) {
23
+ const uint32_t scale = uint32_t(data_a[ib0 + i].scales[itid]);
24
+ sccache1[csel][ix][itid] = FLOAT_TYPE(scale & 0xF);
25
+ sccache2[csel][ix][itid] = FLOAT_TYPE((scale >> 4) & 0xF);
26
+ }
27
+ barrier();
28
+
29
+ if (i >= num_blocks_per_row)
30
+ continue;
31
+ } else {
32
+ const uint32_t scale = uint32_t(data_a[ib0 + i].scales[itid]);
33
+ sccache1[csel][ix][itid] = FLOAT_TYPE(scale & 0xF);
34
+ sccache2[csel][ix][itid] = FLOAT_TYPE((scale >> 4) & 0xF);
35
+ barrier();
36
+ }
37
+
38
+ const uint32_t qs_u32 = uint32_t(data_a_packed16[ib0 + i].qs[q_offset / 2]) | (uint32_t(data_a_packed16[ib0 + i].qs[q_offset / 2 + 8]) << 16);
39
+ const vec4 qs_u32_0 = vec4(unpack8(qs_u32 & 0x03030303));
40
+ const vec4 qs_u32_2 = vec4(unpack8((qs_u32 >> 2) & 0x03030303));
41
+ const vec4 qs_u32_4 = vec4(unpack8((qs_u32 >> 4) & 0x03030303));
42
+ const vec4 qs_u32_6 = vec4(unpack8((qs_u32 >> 6) & 0x03030303));
43
+
44
+ const FLOAT_TYPEV2 dm = vec2(data_a[ib0 + i].dm);
45
+
46
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
47
+ vec2 b0 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 0]);
48
+ vec2 b16 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 8]);
49
+ vec2 b32 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 16]);
50
+ vec2 b48 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 24]);
51
+ vec2 b64 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 32]);
52
+ vec2 b80 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 40]);
53
+ vec2 b96 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 48]);
54
+ vec2 b112 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 56]);
55
+
56
+ FLOAT_TYPE sum1 = FLOAT_TYPE(0.0);
57
+ FLOAT_TYPE sum2 = FLOAT_TYPE(0.0);
58
+ [[unroll]] for (int l = 0; l < 2; ++l) {
59
+ sum1 = fma(FLOAT_TYPE(b0[l]), sccache1[csel][ix][ 8*v_im] * qs_u32_0[l ],
60
+ fma(FLOAT_TYPE(b16[l]), sccache1[csel][ix][1 + 8*v_im] * qs_u32_0[l+2],
61
+ fma(FLOAT_TYPE(b32[l]), sccache1[csel][ix][2 + 8*v_im] * qs_u32_2[l ],
62
+ fma(FLOAT_TYPE(b48[l]), sccache1[csel][ix][3 + 8*v_im] * qs_u32_2[l+2],
63
+ fma(FLOAT_TYPE(b64[l]), sccache1[csel][ix][4 + 8*v_im] * qs_u32_4[l ],
64
+ fma(FLOAT_TYPE(b80[l]), sccache1[csel][ix][5 + 8*v_im] * qs_u32_4[l+2],
65
+ fma(FLOAT_TYPE(b96[l]), sccache1[csel][ix][6 + 8*v_im] * qs_u32_6[l ],
66
+ fma(FLOAT_TYPE(b112[l]), sccache1[csel][ix][7 + 8*v_im] * qs_u32_6[l+2], sum1))))))));
67
+ sum2 = fma(FLOAT_TYPE(b0[l]), sccache2[csel][ix][ 8*v_im],
68
+ fma(FLOAT_TYPE(b16[l]), sccache2[csel][ix][1 + 8*v_im],
69
+ fma(FLOAT_TYPE(b32[l]), sccache2[csel][ix][2 + 8*v_im],
70
+ fma(FLOAT_TYPE(b48[l]), sccache2[csel][ix][3 + 8*v_im],
71
+ fma(FLOAT_TYPE(b64[l]), sccache2[csel][ix][4 + 8*v_im],
72
+ fma(FLOAT_TYPE(b80[l]), sccache2[csel][ix][5 + 8*v_im],
73
+ fma(FLOAT_TYPE(b96[l]), sccache2[csel][ix][6 + 8*v_im],
74
+ fma(FLOAT_TYPE(b112[l]), sccache2[csel][ix][7 + 8*v_im], sum2))))))));
75
+ }
76
+ temp[j][n] = fma(dm.x, sum1, fma(-dm.y, sum2, temp[j][n]));
77
+ }
78
+ }
79
+ }
80
+
81
+ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
82
+ uint a_offset, b_offset, d_offset;
83
+ get_offsets(a_offset, b_offset, d_offset);
84
+
85
+ const uint num_blocks_per_row = p.ncols / QUANT_K;
86
+
87
+ // 16 threads are used to process each block
88
+ const uint it_size = gl_WorkGroupSize.x/16;
89
+ const uint tid = gl_LocalInvocationID.x;
90
+ const uint itid = tid%16; // 0...15
91
+ const uint ix = tid/16;
92
+
93
+ const uint v_im = itid/8; // 0 or 1. 0 computes 0..., 1 computes 128...
94
+ const uint v_in = itid - 8*v_im; // 0...7
95
+
96
+ const uint l0 = 2*v_in; // 0...15
97
+ const uint q_offset = 32*v_im + l0;
98
+ const uint y_offset = 128*v_im + l0;
99
+
100
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
101
+ [[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) {
102
+ temp[j][i] = FLOAT_TYPE(0);
103
+ }
104
+ }
105
+
106
+ const uint nbr_par_th = num_blocks_per_row%it_size;
107
+ const uint nbr_all_th = num_blocks_per_row - nbr_par_th;
108
+ uint i0 = 0;
109
+ [[unroll]] for (; i0 < nbr_all_th; i0 += it_size)
110
+ calc_superblock(a_offset, b_offset, itid, v_im, ix, q_offset, y_offset, i0 + ix, num_blocks_per_row, first_row, num_rows, true);
111
+ calc_superblock(a_offset, b_offset, itid, v_im, ix, q_offset, y_offset, i0 + ix, num_blocks_per_row, first_row, num_rows, false);
112
+
113
+ reduce_result(temp, d_offset, first_row, num_rows, tid);
114
+ }
115
+
116
+ void main() {
117
+ const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z);
118
+
119
+ // do NUM_ROWS at a time, unless there aren't enough remaining rows
120
+ if (first_row + NUM_ROWS <= p.stride_d) {
121
+ compute_outputs(first_row, NUM_ROWS);
122
+ } else {
123
+ if (first_row >= p.stride_d) {
124
+ return;
125
+ }
126
+ compute_outputs(first_row, p.stride_d - first_row);
127
+ }
128
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp ADDED
@@ -0,0 +1,132 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+ #extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
3
+
4
+ #include "mul_mat_vec_base.glsl"
5
+
6
+ layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
7
+
8
+ shared FLOAT_TYPE sccache[2][BLOCK_SIZE/16][2][8];
9
+
10
+ FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];
11
+ uint csel = 0;
12
+
13
+ void calc_superblock(const uint a_offset, const uint b_offset, const uint ix, const uint itid8, const uint v_im, const uint v_im4, const uint v_in, const uint32_t hm_m[4], const uint q_offset, const uint y_offset, const uint s_shift, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows, const bool all_threads) {
14
+ const uint y_idx = i * QUANT_K + y_offset;
15
+
16
+ [[unroll]] for (uint n = 0; n < num_rows; ++n) {
17
+ const uint ib0 = a_offset + (first_row+n)*num_blocks_per_row;
18
+ csel ^= 1;
19
+
20
+ if (!all_threads) { // when we don't have enough blocks to use all threads
21
+ if (i < num_blocks_per_row)
22
+ sccache[csel][ix][v_im][itid8] = FLOAT_TYPE(int8_t(((data_a[ib0+i].scales[itid8] >> v_im4) & 0xF) | (((data_a[ib0+i].scales[itid8%4+8] >> s_shift) & 3) << 4)) - 32);
23
+ barrier();
24
+
25
+ if (i >= num_blocks_per_row)
26
+ continue;
27
+ }
28
+
29
+ const uint32_t hmk = ~(uint32_t(data_a_packed16[ib0 + i].hmask[v_in]) | (uint32_t(data_a_packed16[ib0 + i].hmask[v_in + 8]) << 16));
30
+ const vec4 hmk_0 = vec4(unpack8(((hmk & hm_m[0]) >> ( v_im4)) << 2));
31
+ const vec4 hmk_1 = vec4(unpack8(((hmk & hm_m[1]) >> (1 + v_im4)) << 2));
32
+ const vec4 hmk_2 = vec4(unpack8(((hmk & hm_m[2]) >> (2 + v_im4)) << 2));
33
+ const vec4 hmk_3 = vec4(unpack8(((hmk & hm_m[3]) >> (3 + v_im4)) << 2));
34
+
35
+ // 0, 1, 16, 17
36
+ uint32_t qs_u32 = uint32_t(data_a[ib0 + i].qs[q_offset]) | (uint32_t(data_a[ib0 + i].qs[q_offset + 1]) << 8);
37
+ qs_u32 |= (uint32_t(data_a[ib0 + i].qs[q_offset + 16]) | (uint32_t(data_a[ib0 + i].qs[q_offset + 17]) << 8)) << 16;
38
+ const vec4 qs_u32_0 = vec4(unpack8(qs_u32 & 0x03030303));
39
+ const vec4 qs_u32_2 = vec4(unpack8((qs_u32 >> 2) & 0x03030303));
40
+ const vec4 qs_u32_4 = vec4(unpack8((qs_u32 >> 4) & 0x03030303));
41
+ const vec4 qs_u32_6 = vec4(unpack8((qs_u32 >> 6) & 0x03030303));
42
+
43
+ if (all_threads) {
44
+ sccache[csel][ix][v_im][itid8] = FLOAT_TYPE(int8_t(((data_a[ib0+i].scales[itid8] >> v_im4) & 0xF) | (((data_a[ib0+i].scales[itid8%4+8] >> s_shift) & 3) << 4)) - 32);
45
+ barrier();
46
+ }
47
+
48
+ const FLOAT_TYPE d = FLOAT_TYPE(data_a[ib0 + i].d);
49
+
50
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
51
+ vec2 b0 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 0]);
52
+ vec2 b16 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 8]);
53
+ vec2 b32 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 16]);
54
+ vec2 b48 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 24]);
55
+ vec2 b64 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 32]);
56
+ vec2 b80 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 40]);
57
+ vec2 b96 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 48]);
58
+ vec2 b112 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 56]);
59
+
60
+ FLOAT_TYPE sum = FLOAT_TYPE(0.0);
61
+ [[unroll]] for (int l = 0; l < 2; ++l) {
62
+ sum = fma(FLOAT_TYPE( b0[l]) * sccache[csel][ix][v_im][0], qs_u32_0[l ] - hmk_0[l ],
63
+ fma(FLOAT_TYPE( b16[l]) * sccache[csel][ix][v_im][1], qs_u32_0[l+2] - hmk_0[l+2],
64
+ fma(FLOAT_TYPE( b32[l]) * sccache[csel][ix][v_im][2], qs_u32_2[l ] - hmk_1[l ],
65
+ fma(FLOAT_TYPE( b48[l]) * sccache[csel][ix][v_im][3], qs_u32_2[l+2] - hmk_1[l+2],
66
+ fma(FLOAT_TYPE( b64[l]) * sccache[csel][ix][v_im][4], qs_u32_4[l ] - hmk_2[l ],
67
+ fma(FLOAT_TYPE( b80[l]) * sccache[csel][ix][v_im][5], qs_u32_4[l+2] - hmk_2[l+2],
68
+ fma(FLOAT_TYPE( b96[l]) * sccache[csel][ix][v_im][6], qs_u32_6[l ] - hmk_3[l ],
69
+ fma(FLOAT_TYPE(b112[l]) * sccache[csel][ix][v_im][7], qs_u32_6[l+2] - hmk_3[l+2], sum))))))));
70
+ }
71
+ temp[j][n] = fma(d, sum, temp[j][n]);
72
+ }
73
+ }
74
+ }
75
+
76
+ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
77
+ uint a_offset, b_offset, d_offset;
78
+ get_offsets(a_offset, b_offset, d_offset);
79
+
80
+ const uint num_blocks_per_row = p.ncols / QUANT_K;
81
+
82
+ // 16 threads are used to process each block
83
+ const uint it_size = gl_WorkGroupSize.x/16;
84
+ const uint tid = gl_LocalInvocationID.x;
85
+ const uint itid = tid%16; // 0...15
86
+ const uint ix = tid/16;
87
+ const uint itid8 = itid%8;
88
+
89
+ const uint v_im = itid/8; // 0 or 1. 0 computes 0..., 1 computes 128...
90
+ const uint v_im4 = v_im*4;
91
+ const uint v_in = itid - 8*v_im; // 0...7
92
+
93
+ const uint32_t m = 0x01010101 << (4 * v_im);
94
+ uint32_t hm_m[4];
95
+ [[unroll]] for (uint j = 0; j < 4; ++j)
96
+ hm_m[j] = m << j;
97
+
98
+ const uint l0 = 2*v_in; // 0...15
99
+ const uint q_offset = 32*v_im + l0;
100
+ const uint y_offset = 128*v_im + l0;
101
+
102
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
103
+ [[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) {
104
+ temp[j][i] = FLOAT_TYPE(0);
105
+ }
106
+ }
107
+
108
+ const uint s_shift = v_im4 + 2*(itid8/4);
109
+
110
+ const uint nbr_par_th = num_blocks_per_row%it_size;
111
+ const uint nbr_all_th = num_blocks_per_row - nbr_par_th;
112
+ uint i0 = 0;
113
+ [[unroll]] for (; i0 < nbr_all_th; i0 += it_size)
114
+ calc_superblock(a_offset, b_offset, ix, itid8, v_im, v_im4, v_in, hm_m, q_offset, y_offset, s_shift, i0 + ix, num_blocks_per_row, first_row, num_rows, true);
115
+ calc_superblock(a_offset, b_offset, ix, itid8, v_im, v_im4, v_in, hm_m, q_offset, y_offset, s_shift, i0 + ix, num_blocks_per_row, first_row, num_rows, false);
116
+
117
+ reduce_result(temp, d_offset, first_row, num_rows, tid);
118
+ }
119
+
120
+ void main() {
121
+ const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z);
122
+
123
+ // do NUM_ROWS at a time, unless there aren't enough remaining rows
124
+ if (first_row + NUM_ROWS <= p.stride_d) {
125
+ compute_outputs(first_row, NUM_ROWS);
126
+ } else {
127
+ if (first_row >= p.stride_d) {
128
+ return;
129
+ }
130
+ compute_outputs(first_row, p.stride_d - first_row);
131
+ }
132
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp ADDED
@@ -0,0 +1,134 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+
3
+ #extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
4
+
5
+ #include "mul_mat_vec_base.glsl"
6
+
7
+ layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
8
+
9
+ FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];
10
+
11
+ void calc_superblock(const uint a_offset, const uint b_offset, const uint v_im, const uint q_offset, const uint y_offset, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) {
12
+ const uint y1_idx = i * QUANT_K + y_offset;
13
+ const uint y2_idx = y1_idx + 128;
14
+
15
+ [[unroll]] for (uint n = 0; n < num_rows; ++n) {
16
+ const uint ib0 = a_offset + (first_row+n)*num_blocks_per_row;
17
+ const FLOAT_TYPEV2 dm = FLOAT_TYPEV2(data_a[ib0 + i].dm);
18
+
19
+ const uint32_t scale0_u32 = data_a_packed16[ib0 + i].scales[v_im ];
20
+ const uint32_t scale4_u32 = data_a_packed16[ib0 + i].scales[v_im + 2];
21
+ const uint32_t scale8_u32 = data_a_packed16[ib0 + i].scales[v_im + 4];
22
+
23
+ const uint32_t scale_0_4_l = (scale4_u32 << 16) | scale0_u32;
24
+ const uint32_t scale_0_4_h = (scale_0_4_l & 0xC0C0C0C0) >> 2;
25
+ const vec4 scale_0_4_l_f = vec4(unpack8(scale_0_4_l & 0x3F3F3F3F));
26
+ const vec4 scale8_f = vec4(unpack8((((scale8_u32 << 12) | scale8_u32) & 0x0F0F0F0F) | scale_0_4_h));
27
+
28
+ const FLOAT_TYPE sc0 = scale_0_4_l_f.x;
29
+ const FLOAT_TYPE sc1 = scale_0_4_l_f.y;
30
+ const FLOAT_TYPE sc2 = scale_0_4_l_f.z;
31
+ const FLOAT_TYPE sc3 = scale_0_4_l_f.w;
32
+ const FLOAT_TYPE sc4 = scale8_f.x;
33
+ const FLOAT_TYPE sc5 = scale8_f.y;
34
+ const FLOAT_TYPE sc6 = scale8_f.z;
35
+ const FLOAT_TYPE sc7 = scale8_f.w;
36
+
37
+ const uint32_t qs0_u32 = data_a_packed32[ib0 + i].qs[q_offset / 4];
38
+ const uint32_t qs64_u32 = data_a_packed32[ib0 + i].qs[q_offset / 4 + 16];
39
+
40
+ const uint32_t qs0_u32_lo4 = qs0_u32 & 0x0F0F0F0F;
41
+ const uint32_t qs0_u32_hi4 = (qs0_u32 >> 4) & 0x0F0F0F0F;
42
+ const uint32_t qs64_u32_lo4 = qs64_u32 & 0x0F0F0F0F;
43
+ const uint32_t qs64_u32_hi4 = (qs64_u32 >> 4) & 0x0F0F0F0F;
44
+
45
+ const vec4 qs0_lo4 = vec4(unpack8(qs0_u32_lo4));
46
+ const vec4 qs64_lo4 = vec4(unpack8(qs64_u32_lo4));
47
+ const vec4 qs0_hi4 = vec4(unpack8(qs0_u32_hi4));
48
+ const vec4 qs64_hi4 = vec4(unpack8(qs64_u32_hi4));
49
+
50
+ const FLOAT_TYPE q4_0 = qs0_lo4.x;
51
+ const FLOAT_TYPE q4_1 = qs0_lo4.y;
52
+ const FLOAT_TYPE q4_2 = qs0_lo4.z;
53
+ const FLOAT_TYPE q4_3 = qs0_lo4.w;
54
+ const FLOAT_TYPE q4_4 = qs0_hi4.x;
55
+ const FLOAT_TYPE q4_5 = qs0_hi4.y;
56
+ const FLOAT_TYPE q4_6 = qs0_hi4.z;
57
+ const FLOAT_TYPE q4_7 = qs0_hi4.w;
58
+ const FLOAT_TYPE q4_8 = qs64_lo4.x;
59
+ const FLOAT_TYPE q4_9 = qs64_lo4.y;
60
+ const FLOAT_TYPE q4_10 = qs64_lo4.z;
61
+ const FLOAT_TYPE q4_11 = qs64_lo4.w;
62
+ const FLOAT_TYPE q4_12 = qs64_hi4.x;
63
+ const FLOAT_TYPE q4_13 = qs64_hi4.y;
64
+ const FLOAT_TYPE q4_14 = qs64_hi4.z;
65
+ const FLOAT_TYPE q4_15 = qs64_hi4.w;
66
+
67
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
68
+ vec4 by10 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y1_idx) / 4 ]);
69
+ vec4 by132 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y1_idx) / 4 + 8]);
70
+ vec4 by20 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y2_idx) / 4 ]);
71
+ vec4 by232 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y2_idx) / 4 + 8]);
72
+
73
+ const FLOAT_TYPE sx = fma(FLOAT_TYPE(by10.x), q4_0, fma(FLOAT_TYPE(by10.y), q4_1, fma(FLOAT_TYPE(by10.z), q4_2, FLOAT_TYPE(by10.w) * q4_3)));
74
+ const FLOAT_TYPE sy = fma(FLOAT_TYPE(by132.x), q4_4, fma(FLOAT_TYPE(by132.y), q4_5, fma(FLOAT_TYPE(by132.z), q4_6, FLOAT_TYPE(by132.w) * q4_7)));
75
+ const FLOAT_TYPE sz = fma(FLOAT_TYPE(by20.x), q4_8, fma(FLOAT_TYPE(by20.y), q4_9, fma(FLOAT_TYPE(by20.z), q4_10, FLOAT_TYPE(by20.w) * q4_11)));
76
+ const FLOAT_TYPE sw = fma(FLOAT_TYPE(by232.x), q4_12, fma(FLOAT_TYPE(by232.y), q4_13, fma(FLOAT_TYPE(by232.z), q4_14, FLOAT_TYPE(by232.w) * q4_15)));
77
+ const FLOAT_TYPE smin =
78
+ fma(FLOAT_TYPE(by10.x), sc2, fma(FLOAT_TYPE(by132.x), sc3, fma(FLOAT_TYPE(by20.x), sc6, fma(FLOAT_TYPE(by232.x), sc7,
79
+ fma(FLOAT_TYPE(by10.y), sc2, fma(FLOAT_TYPE(by132.y), sc3, fma(FLOAT_TYPE(by20.y), sc6, fma(FLOAT_TYPE(by232.y), sc7,
80
+ fma(FLOAT_TYPE(by10.z), sc2, fma(FLOAT_TYPE(by132.z), sc3, fma(FLOAT_TYPE(by20.z), sc6, fma(FLOAT_TYPE(by232.z), sc7,
81
+ fma(FLOAT_TYPE(by10.w), sc2, fma(FLOAT_TYPE(by132.w), sc3, fma(FLOAT_TYPE(by20.w), sc6, FLOAT_TYPE(by232.w) * sc7)))))))))))))));
82
+ temp[j][n] = fma(dm.x, fma(sx, sc0, fma(sy, sc1, fma(sz, sc4, sw * sc5))), fma(-dm.y, smin, temp[j][n]));
83
+ }
84
+ }
85
+ }
86
+
87
+ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
88
+ uint a_offset, b_offset, d_offset;
89
+ get_offsets(a_offset, b_offset, d_offset);
90
+
91
+ const uint num_blocks_per_row = p.ncols / QUANT_K;
92
+
93
+ // 16 threads are used to process each block
94
+ const uint it_size = gl_WorkGroupSize.x/16;
95
+ const uint tid = gl_LocalInvocationID.x;
96
+ const uint itid = tid%16; // 0...15
97
+ const uint ix = tid/16;
98
+
99
+ const uint il = itid/4; // 0...3
100
+ const uint ir = itid - 4*il; // 0...3
101
+ const uint n = 4;
102
+
103
+ const uint v_im = il / 2; // 0 or 1. 0 computes 0,32 + 128,160, 1 computes 64,96 + 192,224
104
+ const uint v_in = il % 2;
105
+
106
+ const uint l0 = n * (2 * ir + v_in); // 0...15
107
+ const uint q_offset = 32*v_im + l0;
108
+ const uint y_offset = 64*v_im + l0;
109
+
110
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
111
+ [[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) {
112
+ temp[j][i] = FLOAT_TYPE(0);
113
+ }
114
+ }
115
+
116
+ [[unroll]] for (uint i = ix; i < num_blocks_per_row; i += it_size)
117
+ calc_superblock(a_offset, b_offset, v_im, q_offset, y_offset, i, num_blocks_per_row, first_row, num_rows);
118
+
119
+ reduce_result(temp, d_offset, first_row, num_rows, tid);
120
+ }
121
+
122
+ void main() {
123
+ const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z);
124
+
125
+ // do NUM_ROWS at a time, unless there aren't enough remaining rows
126
+ if (first_row + NUM_ROWS <= p.stride_d) {
127
+ compute_outputs(first_row, NUM_ROWS);
128
+ } else {
129
+ if (first_row >= p.stride_d) {
130
+ return;
131
+ }
132
+ compute_outputs(first_row, p.stride_d - first_row);
133
+ }
134
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp ADDED
@@ -0,0 +1,165 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+
3
+ #extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
4
+
5
+ #include "mul_mat_vec_base.glsl"
6
+
7
+ layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
8
+
9
+ FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];
10
+
11
+ void calc_superblock(const uint a_offset, const uint b_offset, const uint v_im, const uint l0, const uint q_offset, const uint y_offset, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) {
12
+ const uint y1_idx = i * QUANT_K + y_offset;
13
+ const uint y2_idx = y1_idx + 128;
14
+
15
+ [[unroll]] for (uint n = 0; n < num_rows; ++n) {
16
+ const uint ib0 = a_offset + (first_row+n)*num_blocks_per_row;
17
+ const FLOAT_TYPEV2 dm = FLOAT_TYPEV2(data_a[ib0 + i].dm);
18
+
19
+ const uint32_t scale0_u32 = data_a_packed16[ib0 + i].scales[v_im ];
20
+ const uint32_t scale4_u32 = data_a_packed16[ib0 + i].scales[v_im + 2];
21
+ const uint32_t scale8_u32 = data_a_packed16[ib0 + i].scales[v_im + 4];
22
+
23
+ const uint32_t scale_0_4_l = (scale4_u32 << 16) | scale0_u32;
24
+ const uint32_t scale_0_4_h = (scale_0_4_l & 0xC0C0C0C0) >> 2;
25
+ const vec4 scale_0_4_l_f = vec4(unpack8(scale_0_4_l & 0x3F3F3F3F));
26
+ const vec4 scale8_f = vec4(unpack8((((scale8_u32 << 12) | scale8_u32) & 0x0F0F0F0F) | scale_0_4_h));
27
+
28
+ const FLOAT_TYPE sc0 = scale_0_4_l_f.x;
29
+ const FLOAT_TYPE sc1 = scale_0_4_l_f.y;
30
+ const FLOAT_TYPE sc2 = scale_0_4_l_f.z;
31
+ const FLOAT_TYPE sc3 = scale_0_4_l_f.w;
32
+ const FLOAT_TYPE sc4 = scale8_f.x;
33
+ const FLOAT_TYPE sc5 = scale8_f.y;
34
+ const FLOAT_TYPE sc6 = scale8_f.z;
35
+ const FLOAT_TYPE sc7 = scale8_f.w;
36
+
37
+ const uint32_t qs0_16_u32 = uint32_t(data_a_packed16[ib0 + i].qs[q_offset / 2]) | (uint32_t(data_a_packed16[ib0 + i].qs[q_offset / 2 + 8]) << 16);
38
+ const uint32_t qs64_80_u32 = uint32_t(data_a_packed16[ib0 + i].qs[q_offset / 2 + 32]) | (uint32_t(data_a_packed16[ib0 + i].qs[q_offset / 2 + 40]) << 16);
39
+
40
+ uint32_t qs0_16_u32_lo4 = qs0_16_u32 & 0x0F0F0F0F;
41
+ uint32_t qs0_16_u32_hi4 = (qs0_16_u32 >> 4) & 0x0F0F0F0F;
42
+ uint32_t qs64_80_u32_lo4 = qs64_80_u32 & 0x0F0F0F0F;
43
+ uint32_t qs64_80_u32_hi4 = (qs64_80_u32 >> 4) & 0x0F0F0F0F;
44
+
45
+ const uint32_t qh = pack32(u16vec2(data_a_packed16[ib0 + i].qh[l0 / 2], data_a_packed16[ib0 + i].qh[l0 / 2 + 8]));
46
+
47
+ const uint32_t qs0_16_lo4_offset16 = ((qh >> (2*v_im)) & 0x01010101) << 4;
48
+ const uint32_t qs0_16_hi4_offset16 = ((qh >> (2*v_im)) & 0x02020202) << 3;
49
+ const uint32_t qs64_80_lo4_offset16 = ((qh >> (2*v_im)) & 0x10101010);
50
+ const uint32_t qs64_80_hi4_offset16 = ((qh >> (2*v_im)) & 0x20202020) >> 1;
51
+
52
+ qs0_16_u32_lo4 += qs0_16_lo4_offset16;
53
+ qs0_16_u32_hi4 += qs0_16_hi4_offset16;
54
+ qs64_80_u32_lo4 += qs64_80_lo4_offset16;
55
+ qs64_80_u32_hi4 += qs64_80_hi4_offset16;
56
+
57
+ const vec4 qs0_16_lo4 = vec4(unpack8(qs0_16_u32_lo4));
58
+ const vec4 qs64_80_lo4 = vec4(unpack8(qs64_80_u32_lo4));
59
+ const vec4 qs0_16_hi4 = vec4(unpack8(qs0_16_u32_hi4));
60
+ const vec4 qs64_80_hi4 = vec4(unpack8(qs64_80_u32_hi4));
61
+
62
+ const FLOAT_TYPE q4_0 = qs0_16_lo4.x;
63
+ const FLOAT_TYPE q4_1 = qs0_16_lo4.y;
64
+ const FLOAT_TYPE q4_2 = qs0_16_lo4.z;
65
+ const FLOAT_TYPE q4_3 = qs0_16_lo4.w;
66
+ const FLOAT_TYPE q4_4 = qs0_16_hi4.x;
67
+ const FLOAT_TYPE q4_5 = qs0_16_hi4.y;
68
+ const FLOAT_TYPE q4_6 = qs0_16_hi4.z;
69
+ const FLOAT_TYPE q4_7 = qs0_16_hi4.w;
70
+ const FLOAT_TYPE q4_8 = qs64_80_lo4.x;
71
+ const FLOAT_TYPE q4_9 = qs64_80_lo4.y;
72
+ const FLOAT_TYPE q4_10 = qs64_80_lo4.z;
73
+ const FLOAT_TYPE q4_11 = qs64_80_lo4.w;
74
+ const FLOAT_TYPE q4_12 = qs64_80_hi4.x;
75
+ const FLOAT_TYPE q4_13 = qs64_80_hi4.y;
76
+ const FLOAT_TYPE q4_14 = qs64_80_hi4.z;
77
+ const FLOAT_TYPE q4_15 = qs64_80_hi4.w;
78
+
79
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
80
+ vec2 by10 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y1_idx) / 2 ]);
81
+ vec2 by116 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y1_idx) / 2 + 8]);
82
+ vec2 by132 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y1_idx) / 2 + 16]);
83
+ vec2 by148 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y1_idx) / 2 + 24]);
84
+ vec2 by20 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y2_idx) / 2 ]);
85
+ vec2 by216 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y2_idx) / 2 + 8]);
86
+ vec2 by232 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y2_idx) / 2 + 16]);
87
+ vec2 by248 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y2_idx) / 2 + 24]);
88
+
89
+ const FLOAT_TYPE sx =
90
+ fma(FLOAT_TYPE(by10.x), q4_0,
91
+ fma(FLOAT_TYPE(by10.y), q4_1,
92
+ fma(FLOAT_TYPE(by116.x), q4_2,
93
+ FLOAT_TYPE(by116.y) * q4_3)));
94
+ const FLOAT_TYPE sy =
95
+ fma(FLOAT_TYPE(by132.x), q4_4,
96
+ fma(FLOAT_TYPE(by132.y), q4_5,
97
+ fma(FLOAT_TYPE(by148.x), q4_6,
98
+ FLOAT_TYPE(by148.y) * q4_7)));
99
+ const FLOAT_TYPE sz =
100
+ fma(FLOAT_TYPE(by20.x), q4_8,
101
+ fma(FLOAT_TYPE(by20.y), q4_9,
102
+ fma(FLOAT_TYPE(by216.x), q4_10,
103
+ FLOAT_TYPE(by216.y) * q4_11)));
104
+ const FLOAT_TYPE sw =
105
+ fma(FLOAT_TYPE(by232.x), q4_12,
106
+ fma(FLOAT_TYPE(by232.y), q4_13,
107
+ fma(FLOAT_TYPE(by248.x), q4_14,
108
+ FLOAT_TYPE(by248.y) * q4_15)));
109
+ const FLOAT_TYPE smin =
110
+ fma(FLOAT_TYPE(by10.x) + FLOAT_TYPE(by10.y) + FLOAT_TYPE(by116.x) + FLOAT_TYPE(by116.y), sc2,
111
+ fma(FLOAT_TYPE(by132.x) + FLOAT_TYPE(by132.y) + FLOAT_TYPE(by148.x) + FLOAT_TYPE(by148.y), sc3,
112
+ fma(FLOAT_TYPE(by20.x) + FLOAT_TYPE(by20.y) + FLOAT_TYPE(by216.x) + FLOAT_TYPE(by216.y), sc6,
113
+ (FLOAT_TYPE(by232.x) + FLOAT_TYPE(by232.y) + FLOAT_TYPE(by248.x) + FLOAT_TYPE(by248.y)) * sc7)));
114
+ temp[j][n] = fma(dm.x, fma(sx, sc0, fma(sy, sc1, fma(sz, sc4, sw * sc5))), fma(-dm.y, smin, temp[j][n]));
115
+ }
116
+ }
117
+ }
118
+
119
+ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
120
+ uint a_offset, b_offset, d_offset;
121
+ get_offsets(a_offset, b_offset, d_offset);
122
+
123
+ const uint num_blocks_per_row = p.ncols / QUANT_K;
124
+
125
+ // 16 threads are used to process each block
126
+ const uint it_size = gl_WorkGroupSize.x/16;
127
+ const uint tid = gl_LocalInvocationID.x;
128
+ const uint itid = tid%16; // 0...15
129
+ const uint ix = tid/16;
130
+
131
+ const uint il = itid/4; // 0...3
132
+ const uint ir = itid - 4*il; // 0...3
133
+
134
+ const uint v_im = il / 2; // 0 or 1. 0 computes 0,32 + 128,160, 1 computes 64,96 + 192,224
135
+ const uint v_in = il % 2;
136
+
137
+ const uint l0 = 4*ir + 2*v_in; // 0...15
138
+ const uint q_offset = 32*v_im + l0;
139
+ const uint y_offset = 64*v_im + l0;
140
+
141
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
142
+ [[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) {
143
+ temp[j][i] = FLOAT_TYPE(0);
144
+ }
145
+ }
146
+
147
+ [[unroll]] for (uint i = ix; i < num_blocks_per_row; i += it_size)
148
+ calc_superblock(a_offset, b_offset, v_im, l0, q_offset, y_offset, i, num_blocks_per_row, first_row, num_rows);
149
+
150
+ reduce_result(temp, d_offset, first_row, num_rows, tid);
151
+ }
152
+
153
+ void main() {
154
+ const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z);
155
+
156
+ // do NUM_ROWS at a time, unless there aren't enough remaining rows
157
+ if (first_row + NUM_ROWS <= p.stride_d) {
158
+ compute_outputs(first_row, NUM_ROWS);
159
+ } else {
160
+ if (first_row >= p.stride_d) {
161
+ return;
162
+ }
163
+ compute_outputs(first_row, p.stride_d - first_row);
164
+ }
165
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp ADDED
@@ -0,0 +1,130 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+
3
+ #extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
4
+
5
+ #include "mul_mat_vec_base.glsl"
6
+
7
+ layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
8
+
9
+ shared FLOAT_TYPE sccache[2][BLOCK_SIZE/16][16];
10
+
11
+ FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];
12
+ uint csel = 0;
13
+
14
+ void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint ix, const uint ql_offset, const uint qh_offset, const uint s_offset, const uint y_offset, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows, const bool all_threads) {
15
+ const uint y_idx = i * QUANT_K + y_offset;
16
+
17
+ [[unroll]] for (uint n = 0; n < num_rows; ++n) {
18
+ const uint ib0 = a_offset + (first_row+n)*num_blocks_per_row;
19
+ csel ^= 1;
20
+
21
+ if (!all_threads) { // when we don't have enough blocks to use all threads
22
+ if (i < num_blocks_per_row)
23
+ sccache[csel][ix][itid] = FLOAT_TYPE(data_a[ib0 + i].scales[itid]);
24
+ barrier();
25
+
26
+ if (i >= num_blocks_per_row)
27
+ continue;
28
+ }
29
+
30
+ const uint32_t ql0_u32 = uint32_t(data_a_packed16[ib0 + i].ql[ql_offset / 2]) | (uint32_t(data_a_packed16[ib0 + i].ql[ql_offset / 2 + 1]) << 16);
31
+ const uint32_t ql32_u32 = uint32_t(data_a_packed16[ib0 + i].ql[ql_offset / 2 + 16]) | (uint32_t(data_a_packed16[ib0 + i].ql[ql_offset / 2 + 17]) << 16);
32
+
33
+ const uint32_t ql0_u32_lo4 = ql0_u32 & 0x0F0F0F0F;
34
+ const uint32_t ql0_u32_hi4 = (ql0_u32 >> 4) & 0x0F0F0F0F;
35
+ const uint32_t ql32_u32_lo4 = ql32_u32 & 0x0F0F0F0F;
36
+ const uint32_t ql32_u32_hi4 = (ql32_u32 >> 4) & 0x0F0F0F0F;
37
+
38
+ const uint32_t qh_u32 = uint32_t(data_a_packed16[ib0 + i].qh[qh_offset / 2]) | (uint32_t(data_a_packed16[ib0 + i].qh[qh_offset / 2 + 1]) << 16);
39
+ const uint32_t qh0_u32 = (qh_u32 & 0x03030303) << 4;
40
+ const uint32_t qh2_u32 = (qh_u32 & 0x0C0C0C0C) << 2;
41
+ const uint32_t qh4_u32 = (qh_u32 & 0x30303030);
42
+ const uint32_t qh6_u32 = (qh_u32 & 0xC0C0C0C0) >> 2;
43
+
44
+ const uint32_t q0_u32 = ql0_u32_lo4 | qh0_u32;
45
+ const uint32_t q1_u32 = ql32_u32_lo4 | qh2_u32;
46
+ const uint32_t q2_u32 = ql0_u32_hi4 | qh4_u32;
47
+ const uint32_t q3_u32 = ql32_u32_hi4 | qh6_u32;
48
+
49
+ const vec4 q0 = vec4(unpack8(q0_u32)) - 32;
50
+ const vec4 q1 = vec4(unpack8(q1_u32)) - 32;
51
+ const vec4 q2 = vec4(unpack8(q2_u32)) - 32;
52
+ const vec4 q3 = vec4(unpack8(q3_u32)) - 32;
53
+
54
+ if (all_threads) {
55
+ sccache[csel][ix][itid] = FLOAT_TYPE(data_a[ib0 + i].scales[itid]);
56
+ barrier();
57
+ }
58
+
59
+ const FLOAT_TYPE d = FLOAT_TYPE(data_a[ib0 + i].d);
60
+
61
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
62
+ vec4 by0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 ]);
63
+ vec4 by32 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 8]);
64
+ vec4 by64 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 16]);
65
+ vec4 by96 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 24]);
66
+
67
+ FLOAT_TYPE sum[4] = {0, 0, 0, 0};
68
+ [[unroll]] for (uint l = 0; l < 4; ++l) {
69
+ sum[0] = fma(FLOAT_TYPE(by0[l]), q0[l], sum[0]);
70
+ sum[1] = fma(FLOAT_TYPE(by32[l]), q1[l], sum[1]);
71
+ sum[2] = fma(FLOAT_TYPE(by64[l]), q2[l], sum[2]);
72
+ sum[3] = fma(FLOAT_TYPE(by96[l]), q3[l], sum[3]);
73
+ }
74
+ temp[j][n] = fma(fma(sum[0], sccache[csel][ix][s_offset], fma(sum[1], sccache[csel][ix][s_offset + 2], fma(sum[2], sccache[csel][ix][s_offset + 4], sum[3] * sccache[csel][ix][s_offset + 6]))), d, temp[j][n]);
75
+ }
76
+ }
77
+ }
78
+
79
+ void compute_outputs(const uint first_row, const uint num_rows) {
80
+ uint a_offset, b_offset, d_offset;
81
+ get_offsets(a_offset, b_offset, d_offset);
82
+
83
+ const uint num_blocks_per_row = p.ncols / QUANT_K;
84
+
85
+ // 16 threads are used to process each block
86
+ const uint it_size = gl_WorkGroupSize.x/16;
87
+ const uint tid = gl_LocalInvocationID.x;
88
+ const uint itid = tid%16; // 0...15
89
+ const uint ix = tid/16;
90
+
91
+ const uint v_im = itid/8; // 0 or 1. 0 computes 0..., 1 computes 128...
92
+ const uint v_in = itid - 8*v_im; // 0...7
93
+
94
+ const uint l0 = 4 * v_in; // 0, 4, 8, ..., 28
95
+ const uint is = v_in / 4;
96
+
97
+ const uint ql_offset = 64*v_im + l0;
98
+ const uint qh_offset = 32*v_im + l0;
99
+ const uint s_offset = 8*v_im + is;
100
+ const uint y_offset = 128*v_im + l0;
101
+
102
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
103
+ [[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) {
104
+ temp[j][i] = FLOAT_TYPE(0);
105
+ }
106
+ }
107
+
108
+ const uint nbr_par_th = num_blocks_per_row%it_size;
109
+ const uint nbr_all_th = num_blocks_per_row - nbr_par_th;
110
+ uint i0 = 0;
111
+ [[unroll]] for (; i0 < nbr_all_th; i0 += it_size)
112
+ calc_superblock(a_offset, b_offset, itid, ix, ql_offset, qh_offset, s_offset, y_offset, i0 + ix, num_blocks_per_row, first_row, num_rows, true);
113
+ calc_superblock(a_offset, b_offset, itid, ix, ql_offset, qh_offset, s_offset, y_offset, i0 + ix, num_blocks_per_row, first_row, num_rows, false);
114
+
115
+ reduce_result(temp, d_offset, first_row, num_rows, tid);
116
+ }
117
+
118
+ void main() {
119
+ const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z);
120
+
121
+ // do NUM_ROWS at a time, unless there aren't enough remaining rows
122
+ if (first_row + NUM_ROWS <= p.stride_d) {
123
+ compute_outputs(first_row, NUM_ROWS);
124
+ } else {
125
+ if (first_row >= p.stride_d) {
126
+ return;
127
+ }
128
+ compute_outputs(first_row, p.stride_d - first_row);
129
+ }
130
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp ADDED
@@ -0,0 +1,141 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+
3
+ #extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
4
+ #extension GL_EXT_integer_dot_product : require
5
+
6
+ #define MMQ
7
+ #define NEEDS_IQ1S_GRID_GPU
8
+ #define B_TYPE block_q8_1_x4
9
+
10
+ #include "mul_mat_vec_base.glsl"
11
+
12
+ layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
13
+
14
+ #if defined(DATA_A_QUANT_LEGACY) || defined(DATA_A_MXFP4)
15
+ #define K_PER_ITER 8
16
+ #elif defined(DATA_A_QUANT_K)
17
+ #define K_PER_ITER 16
18
+ #elif defined(DATA_A_IQ1_S) || defined(DATA_A_IQ1_M)
19
+ #define K_PER_ITER 32
20
+ #else
21
+ #error unimplemented
22
+ #endif
23
+
24
+ uint a_offset, b_offset, d_offset;
25
+
26
+ int32_t cache_b_qs[K_PER_ITER / 4];
27
+ vec2 cache_b_ds;
28
+
29
+ #include "mul_mat_vecq_funcs.glsl"
30
+
31
+ void iter(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const uint first_row, const uint num_rows, const uint col, const uint b_qs_idx) {
32
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
33
+ // Preload data_b block
34
+ const uint b_block_idx = (j*p.batch_stride_b + col) / QUANT_K_Q8_1 + b_offset;
35
+ const uint b_block_idx_outer = b_block_idx / 4;
36
+ const uint b_block_idx_inner = b_block_idx % 4;
37
+ cache_b_ds = vec2(data_b[b_block_idx_outer].ds[b_block_idx_inner]);
38
+
39
+ #if QUANT_R == 2
40
+ // Assumes K_PER_ITER == 8
41
+ cache_b_qs[0] = data_b[b_block_idx_outer].qs[b_block_idx_inner * 8 + b_qs_idx];
42
+ cache_b_qs[1] = data_b[b_block_idx_outer].qs[b_block_idx_inner * 8 + b_qs_idx + 4];
43
+ #else
44
+ #if K_PER_ITER == 8
45
+ cache_b_qs[0] = data_b[b_block_idx_outer].qs[b_block_idx_inner * 8 + b_qs_idx * 2];
46
+ cache_b_qs[1] = data_b[b_block_idx_outer].qs[b_block_idx_inner * 8 + b_qs_idx * 2 + 1];
47
+ #elif K_PER_ITER == 16
48
+ cache_b_qs[0] = data_b[b_block_idx_outer].qs[b_block_idx_inner * 8 + b_qs_idx * 4 ];
49
+ cache_b_qs[1] = data_b[b_block_idx_outer].qs[b_block_idx_inner * 8 + b_qs_idx * 4 + 1];
50
+ cache_b_qs[2] = data_b[b_block_idx_outer].qs[b_block_idx_inner * 8 + b_qs_idx * 4 + 2];
51
+ cache_b_qs[3] = data_b[b_block_idx_outer].qs[b_block_idx_inner * 8 + b_qs_idx * 4 + 3];
52
+ #elif K_PER_ITER == 32
53
+ cache_b_qs[0] = data_b[b_block_idx_outer].qs[b_block_idx_inner * 8 ];
54
+ cache_b_qs[1] = data_b[b_block_idx_outer].qs[b_block_idx_inner * 8 + 1];
55
+ cache_b_qs[2] = data_b[b_block_idx_outer].qs[b_block_idx_inner * 8 + 2];
56
+ cache_b_qs[3] = data_b[b_block_idx_outer].qs[b_block_idx_inner * 8 + 3];
57
+ cache_b_qs[4] = data_b[b_block_idx_outer].qs[b_block_idx_inner * 8 + 4];
58
+ cache_b_qs[5] = data_b[b_block_idx_outer].qs[b_block_idx_inner * 8 + 5];
59
+ cache_b_qs[6] = data_b[b_block_idx_outer].qs[b_block_idx_inner * 8 + 6];
60
+ cache_b_qs[7] = data_b[b_block_idx_outer].qs[b_block_idx_inner * 8 + 7];
61
+ #else
62
+ #error unimplemented
63
+ #endif
64
+ #endif
65
+
66
+ uint ibi = first_row*p.ncols;
67
+ [[unroll]] for (uint n = 0; n < num_rows; ++n) {
68
+ const uint a_block_idx = (ibi + col)/QUANT_K_Q8_1 + a_offset;
69
+ ibi += p.ncols;
70
+
71
+ temp[j][n] += mmvq_dot_product(a_block_idx, b_qs_idx);
72
+ }
73
+ }
74
+ }
75
+
76
+ void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
77
+ const uint tid = gl_LocalInvocationID.x;
78
+
79
+ get_offsets(a_offset, b_offset, d_offset);
80
+ a_offset *= QUANT_K / QUANT_K_Q8_1;
81
+ b_offset /= QUANT_K_Q8_1;
82
+
83
+ FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];
84
+
85
+ [[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
86
+ [[unroll]] for (uint n = 0; n < num_rows; ++n) {
87
+ temp[j][n] = FLOAT_TYPE(0.0f);
88
+ }
89
+ }
90
+
91
+ const uint col_stride = K_PER_ITER * BLOCK_SIZE;
92
+ uint num_iters = p.ncols / col_stride;
93
+ if (num_iters * col_stride + K_PER_ITER * tid < p.ncols) {
94
+ num_iters++;
95
+ }
96
+
97
+ const uint b_qs_idx = tid % (32 / K_PER_ITER);
98
+ uint col = tid * K_PER_ITER;
99
+ while (num_iters >= 4) {
100
+ // Manually partially unroll the loop
101
+ [[unroll]] for (uint k = 0; k < 4; ++k) {
102
+ iter(temp, first_row, num_rows, col, b_qs_idx);
103
+ col += col_stride;
104
+ }
105
+
106
+ num_iters -= 4;
107
+ }
108
+
109
+ if (num_iters >= 2) {
110
+ // Manually partially unroll the loop
111
+ iter(temp, first_row, num_rows, col, b_qs_idx);
112
+ col += col_stride;
113
+ iter(temp, first_row, num_rows, col, b_qs_idx);
114
+ col += col_stride;
115
+ num_iters -= 2;
116
+ }
117
+
118
+ if (num_iters > 0) {
119
+ iter(temp, first_row, num_rows, col, b_qs_idx);
120
+ }
121
+
122
+ reduce_result(temp, d_offset, first_row, num_rows, tid);
123
+ }
124
+
125
+ void main() {
126
+ const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z);
127
+
128
+ #ifdef NEEDS_INIT_IQ_SHMEM
129
+ init_iq_shmem(gl_WorkGroupSize);
130
+ #endif
131
+
132
+ // do NUM_ROWS at a time, unless there aren't enough remaining rows
133
+ if (first_row + NUM_ROWS <= p.stride_d) {
134
+ compute_outputs(first_row, NUM_ROWS);
135
+ } else {
136
+ if (first_row >= p.stride_d) {
137
+ return;
138
+ }
139
+ compute_outputs(first_row, p.stride_d - first_row);
140
+ }
141
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl ADDED
@@ -0,0 +1,527 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
2
+ #extension GL_EXT_shader_explicit_arithmetic_types_int16 : require
3
+ #extension GL_EXT_shader_explicit_arithmetic_types_int8 : require
4
+
5
+ #include "types.glsl"
6
+
7
+ #if defined(DATA_A_Q4_0) || defined(DATA_A_Q5_0) || defined(DATA_A_Q8_0) || defined(DATA_A_IQ1_S) || defined(DATA_A_IQ2_XXS) || defined(DATA_A_IQ2_XS) || defined(DATA_A_IQ2_S) || defined(DATA_A_IQ3_XXS) || defined(DATA_A_IQ3_S) || defined(DATA_A_IQ4_XS) || defined(DATA_A_IQ4_NL)
8
+ FLOAT_TYPE get_dm(uint ib) {
9
+ return FLOAT_TYPE(data_a[ib].d);
10
+ }
11
+ #endif
12
+
13
+ #if defined(DATA_A_Q4_1) || defined(DATA_A_Q5_1)
14
+ FLOAT_TYPEV2 get_dm(uint ib) {
15
+ return FLOAT_TYPEV2(data_a_packed32[ib].dm);
16
+ }
17
+ #endif
18
+
19
+ #if defined(DATA_A_MXFP4)
20
+ FLOAT_TYPE get_dm(uint ib) {
21
+ return FLOAT_TYPE(e8m0_to_fp32(data_a[ib].e));
22
+ }
23
+ #endif
24
+
25
+ #if defined(DATA_A_Q2_K)
26
+ FLOAT_TYPEV2 get_dm(uint ib) {
27
+ const uint ib_k = ib / 8;
28
+ return FLOAT_TYPEV2(data_a_packed32[ib_k].dm);
29
+ }
30
+ #endif
31
+
32
+ // Each iqs value maps to a 32-bit integer
33
+ #if defined(DATA_A_Q4_0)
34
+ // 2-byte loads for Q4_0 blocks (18 bytes)
35
+ i32vec2 repack(uint ib, uint iqs) {
36
+ const u16vec2 quants = u16vec2(data_a_packed16[ib].qs[iqs * 2 ],
37
+ data_a_packed16[ib].qs[iqs * 2 + 1]);
38
+ const uint32_t vui = pack32(quants);
39
+ return i32vec2( vui & 0x0F0F0F0F,
40
+ (vui >> 4) & 0x0F0F0F0F);
41
+ }
42
+
43
+ FLOAT_TYPE mul_q8_1(const int32_t q_sum, const float da, const vec2 dsb, const int32_t sum_divisor) {
44
+ return FLOAT_TYPE(da * (float(q_sum) * dsb.x - (8 / sum_divisor) * dsb.y));
45
+ }
46
+ #endif
47
+
48
+ #if defined(DATA_A_Q4_1)
49
+ // 4-byte loads for Q4_1 blocks (20 bytes)
50
+ i32vec2 repack(uint ib, uint iqs) {
51
+ const uint32_t vui = data_a_packed32[ib].qs[iqs];
52
+ return i32vec2( vui & 0x0F0F0F0F,
53
+ (vui >> 4) & 0x0F0F0F0F);
54
+ }
55
+
56
+ FLOAT_TYPE mul_q8_1(const int32_t q_sum, const vec2 dma, const vec2 dsb, const int32_t sum_divisor) {
57
+ return FLOAT_TYPE(float(q_sum) * dma.x * dsb.x + dma.y * dsb.y / sum_divisor);
58
+ }
59
+ #endif
60
+
61
+ #if defined(DATA_A_Q5_0)
62
+ // 2-byte loads for Q5_0 blocks (22 bytes)
63
+ i32vec2 repack(uint ib, uint iqs) {
64
+ const u16vec2 quants = u16vec2(data_a_packed16[ib].qs[iqs * 2 ],
65
+ data_a_packed16[ib].qs[iqs * 2 + 1]);
66
+ const uint32_t vui = pack32(quants);
67
+ const int32_t qh = int32_t((uint32_t(data_a_packed16[ib].qh[1]) << 16 | data_a_packed16[ib].qh[0]) >> (4 * iqs));
68
+ const int32_t v0 = int32_t(vui & 0x0F0F0F0F)
69
+ | ((qh & 0xF) * 0x02040810) & 0x10101010; // (0,1,2,3) -> (4,12,20,28)
70
+
71
+ const int32_t v1 = int32_t((vui >> 4) & 0x0F0F0F0F)
72
+ | (((qh >> 16) & 0xF) * 0x02040810) & 0x10101010; // (16,17,18,19) -> (4,12,20,28)
73
+
74
+ return i32vec2(v0, v1);
75
+ }
76
+
77
+ FLOAT_TYPE mul_q8_1(const int32_t q_sum, const float da, const vec2 dsb, const int32_t sum_divisor) {
78
+ return FLOAT_TYPE(da * (float(q_sum) * dsb.x - (16 / sum_divisor) * dsb.y));
79
+ }
80
+ #endif
81
+
82
+ #if defined(DATA_A_Q5_1)
83
+ // 4-byte loads for Q5_1 blocks (24 bytes)
84
+ i32vec2 repack(uint ib, uint iqs) {
85
+ const u16vec2 quants = u16vec2(data_a_packed16[ib].qs[iqs * 2 ],
86
+ data_a_packed16[ib].qs[iqs * 2 + 1]);
87
+ const uint32_t vui = pack32(quants);
88
+ const int32_t qh = int32_t(data_a_packed32[ib].qh >> (4 * iqs));
89
+ const int32_t v0 = int32_t(vui & 0x0F0F0F0F)
90
+ | ((qh & 0xF) * 0x02040810) & 0x10101010; // (0,1,2,3) -> (4,12,20,28)
91
+
92
+ const int32_t v1 = int32_t((vui >> 4) & 0x0F0F0F0F)
93
+ | (((qh >> 16) & 0xF) * 0x02040810) & 0x10101010; // (16,17,18,19) -> (4,12,20,28)
94
+
95
+ return i32vec2(v0, v1);
96
+ }
97
+
98
+ FLOAT_TYPE mul_q8_1(const int32_t q_sum, const vec2 dma, const vec2 dsb, const int32_t sum_divisor) {
99
+ return FLOAT_TYPE(float(q_sum) * dma.x * dsb.x + dma.y * dsb.y / sum_divisor);
100
+ }
101
+ #endif
102
+
103
+ #if defined(DATA_A_Q8_0)
104
+ // 2-byte loads for Q8_0 blocks (34 bytes)
105
+ int32_t repack(uint ib, uint iqs) {
106
+ return pack32(i16vec2(data_a_packed16[ib].qs[iqs * 2 ],
107
+ data_a_packed16[ib].qs[iqs * 2 + 1]));
108
+ }
109
+
110
+ FLOAT_TYPE mul_q8_1(const int32_t q_sum, const float da, const vec2 dsb, const int32_t sum_divisor) {
111
+ return FLOAT_TYPE(float(q_sum) * da * dsb.x);
112
+ }
113
+ #endif
114
+
115
+ #if defined(DATA_A_MXFP4)
116
+ // 1-byte loads for mxfp4 blocks (17 bytes)
117
+ i32vec2 repack(uint ib, uint iqs) {
118
+ const uint32_t qs = pack32(u8vec4(data_a[ib].qs[iqs * 4 ],
119
+ data_a[ib].qs[iqs * 4 + 1],
120
+ data_a[ib].qs[iqs * 4 + 2],
121
+ data_a[ib].qs[iqs * 4 + 3]));
122
+
123
+ const u8vec4 i_a0 = unpack8( qs & 0x0F0F0F0F);
124
+ const u8vec4 i_a1 = unpack8((qs >> 4) & 0x0F0F0F0F);
125
+
126
+ return i32vec2(pack32(i8vec4(kvalues_mxfp4[i_a0.x], kvalues_mxfp4[i_a0.y], kvalues_mxfp4[i_a0.z], kvalues_mxfp4[i_a0.w])),
127
+ pack32(i8vec4(kvalues_mxfp4[i_a1.x], kvalues_mxfp4[i_a1.y], kvalues_mxfp4[i_a1.z], kvalues_mxfp4[i_a1.w])));
128
+ }
129
+
130
+ FLOAT_TYPE mul_q8_1(const int32_t q_sum, const float da, const vec2 dsb, const int32_t sum_divisor) {
131
+ return FLOAT_TYPE(da * dsb.x * float(q_sum) * 0.5);
132
+ }
133
+ #endif
134
+
135
+ #if defined(DATA_A_QUANT_LEGACY) || defined(DATA_A_MXFP4)
136
+ FLOAT_TYPE mmvq_dot_product(const uint ib_a, const uint iqs) {
137
+ int32_t q_sum = 0;
138
+ #if QUANT_R == 2
139
+ const i32vec2 data_a_qs = repack(ib_a, iqs);
140
+ q_sum += dotPacked4x8EXT(data_a_qs.x,
141
+ cache_b_qs[0]);
142
+ q_sum += dotPacked4x8EXT(data_a_qs.y,
143
+ cache_b_qs[1]);
144
+ #else
145
+ int32_t data_a_qs = repack(ib_a, iqs * 2);
146
+ q_sum += dotPacked4x8EXT(data_a_qs,
147
+ cache_b_qs[0]);
148
+ data_a_qs = repack(ib_a, iqs * 2 + 1);
149
+ q_sum += dotPacked4x8EXT(data_a_qs,
150
+ cache_b_qs[1]);
151
+ #endif
152
+
153
+ // 2 quants per call => divide sums by 8/2 = 4
154
+ return mul_q8_1(q_sum, get_dm(ib_a), cache_b_ds, 4);
155
+ }
156
+ #endif
157
+
158
+ #if defined(DATA_A_Q2_K)
159
+ // 4-byte loads for Q2_K blocks (84 bytes)
160
+ i32vec4 repack4(uint ib, uint iqs) {
161
+ const uint ib_k = ib / 8;
162
+ const uint iqs_k = (ib % 8) * 8 + iqs;
163
+
164
+ const uint qs_idx = (iqs_k / 32) * 8 + (iqs_k % 8);
165
+ const uint qs_shift = ((iqs_k % 32) / 8) * 2;
166
+
167
+ return i32vec4((data_a_packed32[ib_k].qs[qs_idx ] >> qs_shift) & 0x03030303,
168
+ (data_a_packed32[ib_k].qs[qs_idx + 1] >> qs_shift) & 0x03030303,
169
+ (data_a_packed32[ib_k].qs[qs_idx + 2] >> qs_shift) & 0x03030303,
170
+ (data_a_packed32[ib_k].qs[qs_idx + 3] >> qs_shift) & 0x03030303);
171
+ }
172
+
173
+ uint8_t get_scale(uint ib, uint iqs) {
174
+ const uint ib_k = ib / 8;
175
+ const uint iqs_k = (ib % 8) * 8 + iqs;
176
+
177
+ return data_a[ib_k].scales[iqs_k / 4];
178
+ }
179
+
180
+ FLOAT_TYPE mmvq_dot_product(const uint ib_a, const uint iqs) {
181
+ int32_t sum_d = 0;
182
+ int32_t sum_m = 0;
183
+
184
+ const i32vec4 qs_a = repack4(ib_a, iqs * 4);
185
+ const uint8_t scale = get_scale(ib_a, iqs * 4);
186
+ const vec2 dm = vec2(get_dm(ib_a));
187
+ const int32_t scale_m = int32_t(scale >> 4) * 0x01010101; // Duplicate 8-bit value across 32-bits.
188
+
189
+ sum_d += dotPacked4x8EXT(qs_a.x, cache_b_qs[0]) * (scale & 0xF);
190
+ sum_m += dotPacked4x8EXT(scale_m, cache_b_qs[0]);
191
+
192
+ sum_d += dotPacked4x8EXT(qs_a.y, cache_b_qs[1]) * (scale & 0xF);
193
+ sum_m += dotPacked4x8EXT(scale_m, cache_b_qs[1]);
194
+
195
+ sum_d += dotPacked4x8EXT(qs_a.z, cache_b_qs[2]) * (scale & 0xF);
196
+ sum_m += dotPacked4x8EXT(scale_m, cache_b_qs[2]);
197
+
198
+ sum_d += dotPacked4x8EXT(qs_a.w, cache_b_qs[3]) * (scale & 0xF);
199
+ sum_m += dotPacked4x8EXT(scale_m, cache_b_qs[3]);
200
+
201
+ return FLOAT_TYPE(float(cache_b_ds.x) * (float(dm.x) * float(sum_d) - float(dm.y) * float(sum_m)));
202
+ }
203
+ #endif
204
+
205
+ #if defined(DATA_A_Q3_K)
206
+ // 2-byte loads for Q3_K blocks (110 bytes)
207
+ i32vec4 repack4(uint ib, uint iqs) {
208
+ const uint ib_k = ib / 8;
209
+ const uint iqs_k = (ib % 8) * 8 + iqs;
210
+
211
+ const uint qs_idx = (iqs_k / 32) * 8 + (iqs_k % 8);
212
+ const uint qs_shift = ((iqs_k % 32) / 8) * 2;
213
+ const uint hm_shift = iqs_k / 8;
214
+
215
+ const uvec4 qs = uvec4( uint32_t(data_a_packed16[ib_k].qs[qs_idx * 2 ]) |
216
+ (uint32_t(data_a_packed16[ib_k].qs[qs_idx * 2 + 1]) << 16),
217
+ uint32_t(data_a_packed16[ib_k].qs[qs_idx * 2 + 2]) |
218
+ (uint32_t(data_a_packed16[ib_k].qs[qs_idx * 2 + 3]) << 16),
219
+ uint32_t(data_a_packed16[ib_k].qs[qs_idx * 2 + 4]) |
220
+ (uint32_t(data_a_packed16[ib_k].qs[qs_idx * 2 + 5]) << 16),
221
+ uint32_t(data_a_packed16[ib_k].qs[qs_idx * 2 + 6]) |
222
+ (uint32_t(data_a_packed16[ib_k].qs[qs_idx * 2 + 7]) << 16));
223
+
224
+ const uvec4 hmask = uvec4( uint32_t(data_a_packed16[ib_k].hmask[iqs * 2 ]) |
225
+ (uint32_t(data_a_packed16[ib_k].hmask[iqs * 2 + 1]) << 16),
226
+ uint32_t(data_a_packed16[ib_k].hmask[iqs * 2 + 2]) |
227
+ (uint32_t(data_a_packed16[ib_k].hmask[iqs * 2 + 3]) << 16),
228
+ uint32_t(data_a_packed16[ib_k].hmask[iqs * 2 + 4]) |
229
+ (uint32_t(data_a_packed16[ib_k].hmask[iqs * 2 + 5]) << 16),
230
+ uint32_t(data_a_packed16[ib_k].hmask[iqs * 2 + 6]) |
231
+ (uint32_t(data_a_packed16[ib_k].hmask[iqs * 2 + 7]) << 16));
232
+
233
+ // bitwise OR to add 4 if hmask is set, subtract later
234
+ const uint vals0 = (( qs.x >> qs_shift) & 0x03030303) |
235
+ (((hmask.x >> hm_shift) & 0x01010101) << 2);
236
+ const uint vals1 = (( qs.y >> qs_shift) & 0x03030303) |
237
+ (((hmask.y >> hm_shift) & 0x01010101) << 2);
238
+ const uint vals2 = (( qs.z >> qs_shift) & 0x03030303) |
239
+ (((hmask.z >> hm_shift) & 0x01010101) << 2);
240
+ const uint vals3 = (( qs.w >> qs_shift) & 0x03030303) |
241
+ (((hmask.w >> hm_shift) & 0x01010101) << 2);
242
+
243
+ // Subtract 4 by twiddling bits rather than using re-packing as mesa
244
+ // compiles repacking poorly.
245
+ return i32vec4(int32_t(((vals0 ^ 0x80808080) - 0x04040404) ^ 0x80808080),
246
+ int32_t(((vals1 ^ 0x80808080) - 0x04040404) ^ 0x80808080),
247
+ int32_t(((vals2 ^ 0x80808080) - 0x04040404) ^ 0x80808080),
248
+ int32_t(((vals3 ^ 0x80808080) - 0x04040404) ^ 0x80808080));
249
+ }
250
+
251
+ float get_d_scale(uint ib, uint iqs) {
252
+ const uint ib_k = ib / 8;
253
+ const uint iqs_k = (ib % 8) * 8 + iqs;
254
+ const uint is = iqs_k / 4;
255
+
256
+ const int8_t scale = int8_t(((data_a[ib_k].scales[is % 8 ] >> (4 * (is / 8))) & 0x0F0F) |
257
+ (((data_a[ib_k].scales[8 + (is % 4)] >> (2 * (is / 4))) & 0x0303) << 4));
258
+ return float(data_a[ib_k].d) * float(scale - 32);
259
+ }
260
+
261
+ FLOAT_TYPE mmvq_dot_product(const uint ib_a, const uint iqs) {
262
+ int32_t q_sum = 0;
263
+
264
+ const i32vec4 qs_a = repack4(ib_a, iqs * 4);
265
+ const float d_scale = get_d_scale(ib_a, iqs * 4);
266
+
267
+ q_sum += dotPacked4x8EXT(qs_a.x, cache_b_qs[0]);
268
+ q_sum += dotPacked4x8EXT(qs_a.y, cache_b_qs[1]);
269
+ q_sum += dotPacked4x8EXT(qs_a.z, cache_b_qs[2]);
270
+ q_sum += dotPacked4x8EXT(qs_a.w, cache_b_qs[3]);
271
+
272
+ return FLOAT_TYPE(float(cache_b_ds.x) * d_scale * float(q_sum));
273
+ }
274
+ #endif
275
+
276
+ #if defined(DATA_A_Q4_K) || defined(DATA_A_Q5_K)
277
+ // 4-byte loads for Q4_K blocks (144 bytes) and Q5_K blocks (176 bytes)
278
+ i32vec4 repack4(uint ib, uint iqs) {
279
+ const uint ib_k = ib / 8;
280
+ const uint iqs_k = (ib % 8) * 8 + iqs;
281
+
282
+ const uint qs_idx = (iqs_k / 16) * 8 + (iqs_k % 8);
283
+ const uint qs_shift = ((iqs_k % 16) / 8) * 4;
284
+
285
+ #if defined(DATA_A_Q4_K)
286
+ const uint32_t vals0 = (data_a_packed32[ib_k].qs[qs_idx ] >> qs_shift) & 0x0F0F0F0F;
287
+ const uint32_t vals1 = (data_a_packed32[ib_k].qs[qs_idx + 1] >> qs_shift) & 0x0F0F0F0F;
288
+ const uint32_t vals2 = (data_a_packed32[ib_k].qs[qs_idx + 2] >> qs_shift) & 0x0F0F0F0F;
289
+ const uint32_t vals3 = (data_a_packed32[ib_k].qs[qs_idx + 3] >> qs_shift) & 0x0F0F0F0F;
290
+
291
+ return i32vec4(vals0, vals1, vals2, vals3);
292
+ #else // defined(DATA_A_Q5_K)
293
+ const uint qh_idx = iqs;
294
+ const uint qh_shift = iqs_k / 8;
295
+
296
+ return i32vec4(((data_a_packed32[ib_k].qs[qs_idx ] >> qs_shift) & 0x0F0F0F0F) |
297
+ (((data_a_packed32[ib_k].qh[qh_idx ] >> qh_shift) & 0x01010101) << 4),
298
+ ((data_a_packed32[ib_k].qs[qs_idx + 1] >> qs_shift) & 0x0F0F0F0F) |
299
+ (((data_a_packed32[ib_k].qh[qh_idx + 1] >> qh_shift) & 0x01010101) << 4),
300
+ ((data_a_packed32[ib_k].qs[qs_idx + 2] >> qs_shift) & 0x0F0F0F0F) |
301
+ (((data_a_packed32[ib_k].qh[qh_idx + 2] >> qh_shift) & 0x01010101) << 4),
302
+ ((data_a_packed32[ib_k].qs[qs_idx + 3] >> qs_shift) & 0x0F0F0F0F) |
303
+ (((data_a_packed32[ib_k].qh[qh_idx + 3] >> qh_shift) & 0x01010101) << 4));
304
+ #endif
305
+ }
306
+
307
+ vec2 get_dm_scale(uint ib, uint iqs) {
308
+ const uint ib_k = ib / 8;
309
+ const uint iqs_k = (ib % 8) * 8 + iqs;
310
+ const uint is = iqs_k / 8;
311
+
312
+ const uvec3 scales = uvec3(data_a_packed32[ib_k].scales[0],
313
+ data_a_packed32[ib_k].scales[1],
314
+ data_a_packed32[ib_k].scales[2]);
315
+ const uint scalesoffs = (is & 3) * 8;
316
+
317
+ const uint scidx0 = (is < 4) ? 0 : 2;
318
+ const uint scidxshift0 = scalesoffs;
319
+ const uint scidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2;
320
+ const uint mbidx0 = (is < 4) ? 1 : 2;
321
+ const uint mbidxshift0 = (is < 4) ? scalesoffs : scalesoffs + 4;
322
+ const uint mbidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2;
323
+
324
+ const uint8_t sc = uint8_t(((scales[scidx0] >> scidxshift0) & 0xF) | ((scales[0] >> scidxshift1) & 0x30));
325
+ const uint8_t mbyte = uint8_t(((scales[mbidx0] >> mbidxshift0) & 0xF) | ((scales[1] >> mbidxshift1) & 0x30));
326
+ u8vec2 scale_dm = u8vec2(sc, mbyte);
327
+
328
+ return FLOAT_TYPEV2(data_a_packed32[ib_k].dm) * FLOAT_TYPEV2(scale_dm);
329
+ }
330
+
331
+ FLOAT_TYPE mmvq_dot_product(const uint ib_a, const uint iqs) {
332
+ int32_t q_sum = 0;
333
+
334
+ const i32vec4 qs_a = repack4(ib_a, iqs * 4);
335
+ const vec2 dm_scale = get_dm_scale(ib_a, iqs * 4);
336
+
337
+ q_sum += dotPacked4x8EXT(qs_a.x, cache_b_qs[0]);
338
+ q_sum += dotPacked4x8EXT(qs_a.y, cache_b_qs[1]);
339
+ q_sum += dotPacked4x8EXT(qs_a.z, cache_b_qs[2]);
340
+ q_sum += dotPacked4x8EXT(qs_a.w, cache_b_qs[3]);
341
+
342
+ return FLOAT_TYPE(float(cache_b_ds.x) * float(dm_scale.x) * float(q_sum) - float(dm_scale.y) * float(cache_b_ds.y / 2));
343
+ }
344
+ #endif
345
+
346
+ #if defined(DATA_A_Q6_K)
347
+ // 2-byte loads for Q6_K blocks (210 bytes)
348
+ i32vec4 repack4(uint ib, uint iqs) {
349
+ const uint ib_k = ib / 8;
350
+ const uint iqs_k = (ib % 8) * 8 + iqs;
351
+
352
+ const uint ql_idx = (iqs_k / 32) * 16 + iqs_k % 16;
353
+ const uint ql_shift = ((iqs_k % 32) / 16) * 4;
354
+
355
+ const uint qh_idx = (iqs_k / 32) * 8 + iqs;
356
+ const uint qh_shift = ((iqs_k % 32) / 8) * 2;
357
+
358
+ const uvec4 ql = uvec4( uint32_t(data_a_packed16[ib_k].ql[ql_idx * 2 ]) |
359
+ (uint32_t(data_a_packed16[ib_k].ql[ql_idx * 2 + 1]) << 16),
360
+ uint32_t(data_a_packed16[ib_k].ql[ql_idx * 2 + 2]) |
361
+ (uint32_t(data_a_packed16[ib_k].ql[ql_idx * 2 + 3]) << 16),
362
+ uint32_t(data_a_packed16[ib_k].ql[ql_idx * 2 + 4]) |
363
+ (uint32_t(data_a_packed16[ib_k].ql[ql_idx * 2 + 5]) << 16),
364
+ uint32_t(data_a_packed16[ib_k].ql[ql_idx * 2 + 6]) |
365
+ (uint32_t(data_a_packed16[ib_k].ql[ql_idx * 2 + 7]) << 16));
366
+
367
+ const uvec4 qh = uvec4( uint32_t(data_a_packed16[ib_k].qh[qh_idx * 2 ]) |
368
+ (uint32_t(data_a_packed16[ib_k].qh[qh_idx * 2 + 1]) << 16),
369
+ uint32_t(data_a_packed16[ib_k].qh[qh_idx * 2 + 2]) |
370
+ (uint32_t(data_a_packed16[ib_k].qh[qh_idx * 2 + 3]) << 16),
371
+ uint32_t(data_a_packed16[ib_k].qh[qh_idx * 2 + 4]) |
372
+ (uint32_t(data_a_packed16[ib_k].qh[qh_idx * 2 + 5]) << 16),
373
+ uint32_t(data_a_packed16[ib_k].qh[qh_idx * 2 + 6]) |
374
+ (uint32_t(data_a_packed16[ib_k].qh[qh_idx * 2 + 7]) << 16));
375
+
376
+ const uint vals0 = (( ql.x >> ql_shift) & 0x0F0F0F0F) |
377
+ (((qh.x >> qh_shift) & 0x03030303) << 4);
378
+ const uint vals1 = (( ql.y >> ql_shift) & 0x0F0F0F0F) |
379
+ (((qh.y >> qh_shift) & 0x03030303) << 4);
380
+ const uint vals2 = (( ql.z >> ql_shift) & 0x0F0F0F0F) |
381
+ (((qh.z >> qh_shift) & 0x03030303) << 4);
382
+ const uint vals3 = (( ql.w >> ql_shift) & 0x0F0F0F0F) |
383
+ (((qh.w >> qh_shift) & 0x03030303) << 4);
384
+
385
+ // Subtract 32 by twiddling bits rather than using re-packing as mesa
386
+ // compiles repacking poorly.
387
+ return i32vec4(int32_t(((vals0 ^ 0x80808080) - 0x20202020) ^ 0x80808080),
388
+ int32_t(((vals1 ^ 0x80808080) - 0x20202020) ^ 0x80808080),
389
+ int32_t(((vals2 ^ 0x80808080) - 0x20202020) ^ 0x80808080),
390
+ int32_t(((vals3 ^ 0x80808080) - 0x20202020) ^ 0x80808080));
391
+ }
392
+
393
+ float get_d_scale(uint ib, uint iqs) {
394
+ const uint ib_k = ib / 8;
395
+ const uint iqs_k = (ib % 8) * 8 + iqs;
396
+ return float(data_a[ib_k].d) * float(data_a[ib_k].scales[iqs_k / 4]);
397
+ }
398
+
399
+ FLOAT_TYPE mmvq_dot_product(const uint ib_a, const uint iqs) {
400
+ int32_t q_sum = 0;
401
+
402
+ const i32vec4 qs_a = repack4(ib_a, iqs * 4);
403
+ const float d_scale = get_d_scale(ib_a, iqs * 4);
404
+
405
+ q_sum += dotPacked4x8EXT(qs_a.x, cache_b_qs[0]);
406
+ q_sum += dotPacked4x8EXT(qs_a.y, cache_b_qs[1]);
407
+ q_sum += dotPacked4x8EXT(qs_a.z, cache_b_qs[2]);
408
+ q_sum += dotPacked4x8EXT(qs_a.w, cache_b_qs[3]);
409
+
410
+ return FLOAT_TYPE(float(cache_b_ds.x) * float(d_scale) * float(q_sum));
411
+ }
412
+ #endif
413
+
414
+ #if defined(DATA_A_IQ1_S)
415
+ void repack8(uint ib, uint iqs, out i32vec4 out0, out i32vec4 out1) {
416
+ const uint ib32 = iqs / 32;
417
+
418
+ const uint qh = data_a[ib].qh[ib32];
419
+
420
+ const uint qs16_0 = data_a_packed16[ib].qs[(4 * ib32 + 0) / 2];
421
+ const uint qs16_1 = data_a_packed16[ib].qs[(4 * ib32 + 2) / 2];
422
+
423
+ const uint qs0 = qs16_0 & 0xFF;
424
+ const uint qs1 = qs16_0 >> 8;
425
+ const uint qs2 = qs16_1 & 0xFF;
426
+ const uint qs3 = qs16_1 >> 8;
427
+
428
+ const uint hi0 = bitfieldExtract(qh, 3 * int(0), 3);
429
+ const uint hi1 = bitfieldExtract(qh, 3 * int(1), 3);
430
+ const uint hi2 = bitfieldExtract(qh, 3 * int(2), 3);
431
+ const uint hi3 = bitfieldExtract(qh, 3 * int(3), 3);
432
+
433
+ const int32_t grid0 = int32_t(iq1s_grid_gpu[qs0 | (hi0 << 8)]);
434
+ const int32_t grid1 = int32_t(iq1s_grid_gpu[qs1 | (hi1 << 8)]);
435
+ const int32_t grid2 = int32_t(iq1s_grid_gpu[qs2 | (hi2 << 8)]);
436
+ const int32_t grid3 = int32_t(iq1s_grid_gpu[qs3 | (hi3 << 8)]);
437
+
438
+ out0 = i32vec4((grid0 >> 0) & 0x0F0F0F0F,
439
+ (grid0 >> 4) & 0x0F0F0F0F,
440
+ (grid1 >> 0) & 0x0F0F0F0F,
441
+ (grid1 >> 4) & 0x0F0F0F0F);
442
+ out1 = i32vec4((grid2 >> 0) & 0x0F0F0F0F,
443
+ (grid2 >> 4) & 0x0F0F0F0F,
444
+ (grid3 >> 0) & 0x0F0F0F0F,
445
+ (grid3 >> 4) & 0x0F0F0F0F);
446
+ }
447
+
448
+ vec2 get_dm(uint ib, uint iqs) {
449
+ const uint ib32 = iqs / 32;
450
+
451
+ const uint qh = data_a[ib].qh[ib32];
452
+ const float delta = ((qh & 0x8000) != 0) ? -IQ1S_DELTA : IQ1S_DELTA;
453
+
454
+ const float d = float(data_a[ib].d);
455
+ const float dl = d * float(2 * bitfieldExtract(qh, 12, 3) + 1);
456
+
457
+ // the -1 cancels out the bias in iq1s_grid_gpu
458
+ return FLOAT_TYPEV2(dl, dl * (delta - 1));
459
+ }
460
+
461
+ FLOAT_TYPE mmvq_dot_product(const uint ib_a, const uint iqs) {
462
+ int32_t q_sum = 0;
463
+
464
+ const uint ib_k = ib_a / 8;
465
+ const uint iqs_k = (ib_a % 8) * 32 + iqs * 32;
466
+
467
+ i32vec4 qs_a0;
468
+ i32vec4 qs_a1;
469
+ repack8(ib_k, iqs_k, qs_a0, qs_a1);
470
+
471
+ const vec2 dm = get_dm(ib_k, iqs_k);
472
+
473
+ q_sum += dotPacked4x8EXT(qs_a0.x, cache_b_qs[0]);
474
+ q_sum += dotPacked4x8EXT(qs_a0.y, cache_b_qs[1]);
475
+ q_sum += dotPacked4x8EXT(qs_a0.z, cache_b_qs[2]);
476
+ q_sum += dotPacked4x8EXT(qs_a0.w, cache_b_qs[3]);
477
+ q_sum += dotPacked4x8EXT(qs_a1.x, cache_b_qs[4]);
478
+ q_sum += dotPacked4x8EXT(qs_a1.y, cache_b_qs[5]);
479
+ q_sum += dotPacked4x8EXT(qs_a1.z, cache_b_qs[6]);
480
+ q_sum += dotPacked4x8EXT(qs_a1.w, cache_b_qs[7]);
481
+
482
+ return FLOAT_TYPE(float(cache_b_ds.x) * float(dm.x) * float(q_sum) + float(dm.y) * float(cache_b_ds.y));
483
+ }
484
+ #endif
485
+
486
+ #if defined(DATA_A_IQ1_M)
487
+ FLOAT_TYPE mmvq_dot_product(const uint ib_a, const uint iqs) {
488
+ const uint ib_k = ib_a / 8;
489
+ const uint iqs_k = (ib_a % 8) * 32 + iqs * 32;
490
+
491
+ const uint ib32 = iqs_k / 32;
492
+ const uint ib64 = ib32 / 2;
493
+
494
+ const uint16_t[4] scales = data_a[ib_k].scales;
495
+ const u16vec4 s = u16vec4(scales[0], scales[1], scales[2], scales[3]) >> 12;
496
+ const float d = float(unpackHalf2x16(s.x | (s.y << 4) | (s.z << 8) | (s.w << 12)).x);
497
+
498
+ const uint qs32 = data_a_packed32[ib_k].qs[ib32];
499
+ const uint qh16 = data_a_packed16[ib_k].qh[ib32];
500
+
501
+ float sum = 0;
502
+ const uint sc = data_a[ib_k].scales[ib64];
503
+ [[unroll]] for (int l = 0; l < 4; ++l) {
504
+ const uint ib16 = 2 * ib32 + l / 2;
505
+ const float dl = d * (2 * bitfieldExtract(sc, 3 * int(ib16 & 3), 3) + 1);
506
+ const uint qh = qh16 >> (4 * l);
507
+ const uint qs = (qs32 >> (8 * l)) & 0xFF;
508
+ const float delta = ((qh & 8) != 0) ? -IQ1M_DELTA : IQ1M_DELTA;
509
+
510
+ const int32_t grid = int32_t(iq1s_grid_gpu[qs | ((qh & 7) << 8)]);
511
+
512
+ int32_t q_sum = 0;
513
+ q_sum += dotPacked4x8EXT((grid >> 0) & 0x0F0F0F0F, cache_b_qs[2 * l + 0]);
514
+ q_sum += dotPacked4x8EXT((grid >> 4) & 0x0F0F0F0F, cache_b_qs[2 * l + 1]);
515
+
516
+ int32_t y_sum = 0;
517
+ y_sum += dotPacked4x8EXT(int(0x01010101), cache_b_qs[2 * l + 0]);
518
+ y_sum += dotPacked4x8EXT(int(0x01010101), cache_b_qs[2 * l + 1]);
519
+
520
+ // the -1 cancels out the bias in iq1s_grid_gpu
521
+ sum += dl * (q_sum + y_sum * (delta - 1));
522
+ }
523
+ sum *= float(cache_b_ds.x);
524
+
525
+ return sum;
526
+ }
527
+ #endif
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp ADDED
@@ -0,0 +1,466 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+
3
+ #extension GL_EXT_control_flow_attributes : enable
4
+ #extension GL_EXT_shader_16bit_storage : require
5
+
6
+ #ifdef FLOAT16
7
+ #extension GL_EXT_shader_explicit_arithmetic_types_float16 : require
8
+ #endif
9
+ #if defined(DATA_A_IQ1_M)
10
+ #extension GL_EXT_shader_explicit_arithmetic_types_int16 : require
11
+ #endif
12
+
13
+ #if defined(DATA_A_BF16) && defined(COOPMAT)
14
+ #extension GL_EXT_bfloat16 : enable
15
+ #endif
16
+
17
+ #ifdef COOPMAT
18
+ #extension GL_KHR_cooperative_matrix : enable
19
+ #extension GL_KHR_memory_scope_semantics : enable
20
+ #endif
21
+
22
+ #if defined(COOPMAT) || defined(MUL_MAT_ID_USE_SUBGROUPS)
23
+ #extension GL_KHR_shader_subgroup_basic : enable
24
+ #extension GL_KHR_shader_subgroup_ballot : enable
25
+ #endif
26
+
27
+ #ifdef MUL_MAT_ID
28
+ #extension GL_EXT_shader_explicit_arithmetic_types_int16 : require
29
+ #endif
30
+
31
+ #include "types.glsl"
32
+ #include "dot_product_funcs.glsl"
33
+
34
+ #ifndef LOAD_VEC_A
35
+ #define LOAD_VEC_A 1
36
+ #endif
37
+ #ifndef LOAD_VEC_B
38
+ #define LOAD_VEC_B 1
39
+ #endif
40
+
41
+ layout (constant_id = 11) const uint ALIGNED = 0;
42
+
43
+ #if !defined(TO_FLOAT_TYPE)
44
+ #define TO_FLOAT_TYPE FLOAT_TYPE
45
+ #endif
46
+
47
+ layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
48
+
49
+ layout (binding = 0) readonly buffer A {A_TYPE data_a[];};
50
+ #if defined(DATA_A_F32)
51
+ layout (binding = 0) readonly buffer A_SCALAR {float data_a_scalar[];};
52
+ #elif defined(DATA_A_F16)
53
+ layout (binding = 0) readonly buffer A_SCALAR {float16_t data_a_scalar[];};
54
+ #elif defined(DATA_A_BF16)
55
+ layout (binding = 0) readonly buffer A_SCALAR {uint16_t data_a_scalar[];};
56
+ #endif
57
+ #if defined(A_TYPE_PACKED16)
58
+ layout (binding = 0) readonly buffer A_PACKED16 {A_TYPE_PACKED16 data_a_packed16[];};
59
+ #endif
60
+ #if defined(A_TYPE_PACKED32)
61
+ layout (binding = 0) readonly buffer A_PACKED32 {A_TYPE_PACKED32 data_a_packed32[];};
62
+ #endif
63
+
64
+ layout (binding = 1) readonly buffer B {B_TYPE data_b[];};
65
+ layout (binding = 1) readonly buffer B_SCALAR {B_TYPE_SCALAR data_b_scalar[];};
66
+ layout (binding = 2) writeonly buffer D {D_TYPE data_d[];};
67
+
68
+ #ifdef MUL_MAT_ID
69
+ layout (binding = 3) readonly buffer IDS {int data_ids[];};
70
+ layout (binding = 4) readonly buffer Counts {int data_expert_count[];};
71
+ #endif
72
+
73
+ layout (push_constant) uniform parameter
74
+ {
75
+ uint M;
76
+ uint N;
77
+ uint K;
78
+ uint stride_a;
79
+ uint stride_b;
80
+ uint stride_d;
81
+
82
+ uint batch_stride_a;
83
+ uint batch_stride_b;
84
+ uint batch_stride_d;
85
+
86
+ #ifdef MUL_MAT_ID
87
+ uint nei0;
88
+ uint nei1;
89
+ uint nbi1;
90
+ uint ne11;
91
+ #else
92
+ uint base_work_group_z;
93
+ uint num_batches;
94
+ uint k_split;
95
+ uint ne02;
96
+ uint ne12;
97
+ uint broadcast2;
98
+ uint broadcast3;
99
+ #endif
100
+ } p;
101
+
102
+ layout (constant_id = 0) const uint BLOCK_SIZE = 64;
103
+ layout (constant_id = 1) const uint BM = 64;
104
+ layout (constant_id = 2) const uint BN = 64;
105
+ layout (constant_id = 4) const uint WM = 32;
106
+ layout (constant_id = 5) const uint WN = 32;
107
+ layout (constant_id = 6) const uint WMITER = 2;
108
+ layout (constant_id = 7) const uint TM = 4;
109
+ layout (constant_id = 8) const uint TN = 2;
110
+ layout (constant_id = 9) const uint TK = 1; // Only needed for coopmat
111
+ layout (constant_id = 10) const uint WARP = 32;
112
+
113
+ #if defined(DATA_A_F32) || defined(DATA_A_F16)
114
+ #define BK 32
115
+ #define BK_STEP 4
116
+ #else
117
+ layout (constant_id = 3) const uint BK = 16; // Assumed to be 32 if working with a quant
118
+ #define BK_STEP 2
119
+ #endif
120
+
121
+ #ifdef COOPMAT
122
+ #define SHMEM_STRIDE (BK / 2 + 4)
123
+ #else
124
+ #define SHMEM_STRIDE (BK / 2 + 1)
125
+ #endif
126
+
127
+ shared FLOAT_TYPEV2 buf_a[BM * SHMEM_STRIDE];
128
+ shared FLOAT_TYPEV2 buf_b[BN * SHMEM_STRIDE];
129
+
130
+ #define NUM_WARPS (BLOCK_SIZE / WARP)
131
+
132
+ #ifdef COOPMAT
133
+ shared ACC_TYPE coopmat_stage[TM * TN * NUM_WARPS];
134
+ #endif
135
+
136
+ #include "mul_mm_id_funcs.glsl"
137
+ #include "mul_mm_funcs.glsl"
138
+
139
+ void main() {
140
+ const uint ic = gl_WorkGroupID.y;
141
+
142
+ #ifdef MUL_MAT_ID
143
+ const uint expert_idx = gl_WorkGroupID.z;
144
+ if (ic * BN >= data_expert_count[expert_idx]) {
145
+ return;
146
+ }
147
+ #endif
148
+ #ifdef NEEDS_INIT_IQ_SHMEM
149
+ init_iq_shmem(gl_WorkGroupSize);
150
+ #endif
151
+
152
+ #ifndef MUL_MAT_ID
153
+ const uint batch_idx = gl_WorkGroupID.z + p.base_work_group_z;
154
+
155
+ const uint i13 = batch_idx / p.ne12;
156
+ const uint i12 = batch_idx % p.ne12;
157
+
158
+ const uint i03 = i13 / p.broadcast3;
159
+ const uint i02 = i12 / p.broadcast2;
160
+
161
+ const uint batch_idx_a = i03 * p.ne02 + i02;
162
+ #endif
163
+
164
+ const uint blocks_m = (p.M + BM - 1) / BM;
165
+ const uint ir = gl_WorkGroupID.x % blocks_m;
166
+ const uint ik = gl_WorkGroupID.x / blocks_m;
167
+
168
+ const uint WNITER = (WM * WN) / (WARP * TM * TN * WMITER);
169
+ const uint WSUBM = WM / WMITER;
170
+ const uint WSUBN = WN / WNITER;
171
+
172
+ #ifdef COOPMAT
173
+ const uint warp_i = gl_SubgroupID;
174
+
175
+ const uint tiw = gl_SubgroupInvocationID;
176
+
177
+ const uint cms_per_row = WM / TM;
178
+ const uint cms_per_col = WN / TN;
179
+
180
+ const uint storestride = WARP / TM;
181
+ const uint store_r = tiw % TM;
182
+ const uint store_c = tiw / TM;
183
+ #else
184
+ const uint warp_i = gl_LocalInvocationID.x / WARP;
185
+
186
+ const uint tiw = gl_LocalInvocationID.x % WARP;
187
+
188
+ const uint tiwr = tiw % (WSUBM / TM);
189
+ const uint tiwc = tiw / (WSUBM / TM);
190
+ #endif
191
+
192
+ const uint warp_r = warp_i % (BM / WM);
193
+ const uint warp_c = warp_i / (BM / WM);
194
+
195
+ #if defined(DATA_A_F32) || defined(DATA_A_F16) || defined(DATA_A_BF16)
196
+ const uint LOAD_VEC_A_EFF = (ALIGNED != 0) ? LOAD_VEC_A : 1;
197
+ const uint LOAD_VEC_BATCH_A = (ALIGNED != 0) ? 1 : 2;
198
+ #else
199
+ const uint LOAD_VEC_A_EFF = LOAD_VEC_A;
200
+ const uint LOAD_VEC_BATCH_A = 1;
201
+ #endif
202
+ const uint LOAD_VEC_B_EFF = (ALIGNED != 0) ? LOAD_VEC_B : 1;
203
+ const uint LOAD_VEC_BATCH_B = (ALIGNED != 0) ? 1 : 2;
204
+
205
+ const uint loadr_a = gl_LocalInvocationID.x % (BK / LOAD_VEC_A_EFF / LOAD_VEC_BATCH_A);
206
+ const uint loadc_a = gl_LocalInvocationID.x / (BK / LOAD_VEC_A_EFF / LOAD_VEC_BATCH_A);
207
+ const uint loadr_b = gl_LocalInvocationID.x % (BK / LOAD_VEC_B_EFF / LOAD_VEC_BATCH_B);
208
+ const uint loadc_b = gl_LocalInvocationID.x / (BK / LOAD_VEC_B_EFF / LOAD_VEC_BATCH_B);
209
+
210
+ const uint loadstride_a = gl_WorkGroupSize.x * LOAD_VEC_A_EFF * LOAD_VEC_BATCH_A / BK;
211
+ const uint loadstride_b = gl_WorkGroupSize.x * LOAD_VEC_B_EFF * LOAD_VEC_BATCH_B / BK;
212
+
213
+ #ifdef MUL_MAT_ID
214
+ #ifdef MUL_MAT_ID_USE_SUBGROUPS
215
+ if (bitCount(p.nei0) == 1) {
216
+ load_row_ids(expert_idx, true, ic);
217
+ } else {
218
+ load_row_ids(expert_idx, false, ic);
219
+ }
220
+ #else
221
+ _ne1 = 0;
222
+ for (uint ii1 = 0; ii1 < p.nei1 && _ne1 < (ic + 1) * BN; ii1++) {
223
+ for (uint ii0 = 0; ii0 < p.nei0 && _ne1 < (ic + 1) * BN; ii0++) {
224
+ if (data_ids[ii1*p.nbi1 + ii0] == expert_idx) {
225
+ if (_ne1 >= ic * BN) {
226
+ row_ids[_ne1 - ic * BN] = u16vec2(ii0, ii1);
227
+ }
228
+ _ne1++;
229
+ }
230
+ }
231
+ }
232
+
233
+ barrier();
234
+ #endif
235
+
236
+ // Workgroup has no work
237
+ if (ic * BN >= _ne1) return;
238
+ #endif
239
+
240
+ #ifdef MUL_MAT_ID
241
+ const uint start_k = 0;
242
+ const uint end_k = p.K;
243
+ #else
244
+ const uint start_k = ik * p.k_split;
245
+ const uint end_k = min(p.K, (ik + 1) * p.k_split);
246
+ #endif
247
+
248
+ uint pos_a =
249
+ #ifdef MUL_MAT_ID
250
+ expert_idx * (p.batch_stride_a / LOAD_VEC_A_EFF) +
251
+ #else
252
+ batch_idx_a * (p.batch_stride_a / LOAD_VEC_A_EFF) +
253
+ #endif
254
+ (ir * BM * p.stride_a + start_k) / LOAD_VEC_A_EFF;
255
+ #ifdef MUL_MAT_ID
256
+ uint pos_b = 0;
257
+ #else
258
+ uint pos_b = (batch_idx * p.batch_stride_b + ic * BN * p.stride_b + start_k) / LOAD_VEC_B_EFF;
259
+ #endif
260
+
261
+ #ifdef COOPMAT
262
+ coopmat<FLOAT_TYPE, gl_ScopeSubgroup, TM, TK, gl_MatrixUseA> cache_a;
263
+ coopmat<FLOAT_TYPE, gl_ScopeSubgroup, TK, TN, gl_MatrixUseB> cache_b;
264
+ coopmat<ACC_TYPE, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator> sums[cms_per_row * cms_per_col];
265
+
266
+ [[unroll]] for (uint i = 0; i < cms_per_row * cms_per_col; i++) {
267
+ sums[i] = coopmat<ACC_TYPE, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator>(0.0f);
268
+ }
269
+ #else
270
+ ACC_TYPEV2 sums[WMITER * TM * WNITER * TN/2];
271
+ #if defined(DATA_A_F32) || defined(DATA_A_F16)
272
+ FLOAT_TYPEV4 cache_a[WMITER * TM];
273
+ FLOAT_TYPEV4 cache_b;
274
+ #else
275
+ FLOAT_TYPEV2 cache_a[WMITER * TM];
276
+ FLOAT_TYPEV2 cache_b;
277
+ #endif
278
+
279
+ [[unroll]] for (uint i = 0; i < WMITER*TM*WNITER*TN/2; i++) {
280
+ sums[i] = ACC_TYPEV2(0.0f, 0.0f);
281
+ }
282
+ #endif
283
+
284
+ for (uint block = start_k; block < end_k; block += BK) {
285
+ [[unroll]] for (uint l = 0; l < BM; l += loadstride_a) {
286
+ load_a_to_shmem(pos_a, loadr_a, loadc_a + l, ir * BM + loadc_a + l, block, end_k);
287
+ }
288
+ [[unroll]] for (uint l = 0; l < BN; l += loadstride_b) {
289
+ #if !defined(MUL_MAT_ID)
290
+ load_b_to_shmem(pos_b, loadr_b, loadc_b + l, ic * BN + loadc_b + l, block, end_k);
291
+ #else
292
+ load_b_to_shmem(pos_b, loadr_b, loadc_b + l, ic, _ne1, block, end_k);
293
+ #endif
294
+ }
295
+
296
+ barrier();
297
+
298
+ pos_a += BK / LOAD_VEC_A_EFF;
299
+ pos_b += BK / LOAD_VEC_B_EFF;
300
+
301
+ #ifdef COOPMAT
302
+ [[unroll]] for (uint i = 0; i < BK; i += TK) {
303
+ [[unroll]] for (uint cm_row = 0; cm_row < cms_per_row; cm_row++) {
304
+ // Load from shared into cache
305
+ coopMatLoad(cache_a, buf_a, (warp_r * WM + cm_row * TM) * SHMEM_STRIDE + i / 2, SHMEM_STRIDE, gl_CooperativeMatrixLayoutRowMajor);
306
+
307
+ [[unroll]] for (uint cm_col = 0; cm_col < cms_per_col; cm_col++) {
308
+ coopMatLoad(cache_b, buf_b, (warp_c * WN + cm_col * TN) * SHMEM_STRIDE + i / 2, SHMEM_STRIDE, gl_CooperativeMatrixLayoutColumnMajor);
309
+
310
+ sums[cm_col * cms_per_row + cm_row] = coopMatMulAdd(cache_a, cache_b, sums[cm_col * cms_per_row + cm_row]);
311
+ }
312
+ }
313
+ }
314
+ #else
315
+ [[unroll]] for (uint i = 0; i < BK / BK_STEP; i++) {
316
+ // Load from shared into cache
317
+ [[unroll]] for (uint wsir = 0; wsir < WMITER; wsir++) {
318
+ [[unroll]] for (uint j = 0; j < TM; j++) {
319
+ #if defined(DATA_A_F32) || defined(DATA_A_F16)
320
+ cache_a[wsir * TM + j].xy = buf_a[(warp_r * WM + wsir * WSUBM + tiwr * TM + j) * SHMEM_STRIDE + 2 * i ];
321
+ cache_a[wsir * TM + j].zw = buf_a[(warp_r * WM + wsir * WSUBM + tiwr * TM + j) * SHMEM_STRIDE + 2 * i + 1];
322
+ #else
323
+ cache_a[wsir * TM + j] = buf_a[(warp_r * WM + wsir * WSUBM + tiwr * TM + j) * SHMEM_STRIDE + i];
324
+ #endif
325
+ }
326
+ }
327
+
328
+ [[unroll]] for (uint wsic = 0; wsic < WNITER; wsic++) {
329
+ [[unroll]] for (uint cc = 0; cc < TN; cc++) {
330
+ #if defined(DATA_A_F32) || defined(DATA_A_F16)
331
+ cache_b.xy = buf_b[(warp_c * WN + wsic * WSUBN + tiwc * TN + cc) * SHMEM_STRIDE + 2 * i ];
332
+ cache_b.zw = buf_b[(warp_c * WN + wsic * WSUBN + tiwc * TN + cc) * SHMEM_STRIDE + 2 * i + 1];
333
+ #else
334
+ cache_b = buf_b[(warp_c * WN + wsic * WSUBN + tiwc * TN + cc) * SHMEM_STRIDE + i];
335
+ #endif
336
+
337
+ [[unroll]] for (uint wsir = 0; wsir < WMITER; wsir++) {
338
+ [[unroll]] for (uint cr = 0; cr < TM / 2; cr++) {
339
+ // [WNITER][TN][WMITER][TM / 2] -> [wsic][cc][wsir][cr]
340
+ const uint sums_idx = (wsic * TN + cc) * WMITER * (TM / 2) + wsir * (TM / 2) + cr;
341
+ sums[sums_idx].x = dot_product(cache_a[wsir * TM + 2 * cr ], cache_b, sums[sums_idx].x);
342
+ sums[sums_idx].y = dot_product(cache_a[wsir * TM + 2 * cr + 1], cache_b, sums[sums_idx].y);
343
+ }
344
+ }
345
+ }
346
+ }
347
+
348
+ }
349
+ #endif
350
+
351
+ barrier();
352
+ }
353
+
354
+ #if defined(ACC_TYPE_MAX)
355
+ #ifdef COOPMAT
356
+ [[unroll]] for (uint j = 0; j < cms_per_row * cms_per_col; j++) {
357
+ [[unroll]] for (uint i = 0; i < sums[j].length(); ++i) {
358
+ sums[j][i] = clamp(sums[j][i], -ACC_TYPE_MAX, ACC_TYPE_MAX);
359
+ }
360
+ }
361
+ #else
362
+ [[unroll]] for (uint i = 0; i < WMITER*TM*WNITER*TN/2; i++) {
363
+ sums[i].x = clamp(sums[i].x, -ACC_TYPE_MAX, ACC_TYPE_MAX);
364
+ sums[i].y = clamp(sums[i].y, -ACC_TYPE_MAX, ACC_TYPE_MAX);
365
+ }
366
+ #endif
367
+ #endif
368
+
369
+ const uint dr = ir * BM + warp_r * WM;
370
+ const uint dc = ic * BN + warp_c * WN;
371
+
372
+ #ifndef MUL_MAT_ID
373
+ const uint offsets = batch_idx * p.batch_stride_d + ik * p.batch_stride_d * p.num_batches;
374
+ #endif
375
+
376
+ #ifdef COOPMAT
377
+ #ifdef MUL_MAT_ID
378
+ [[unroll]] for (uint cm_row = 0; cm_row < cms_per_row; cm_row++) {
379
+ [[unroll]] for (uint cm_col = 0; cm_col < cms_per_col; cm_col++) {
380
+ coopMatStore(sums[cm_col * cms_per_row + cm_row], coopmat_stage, warp_i * TM * TN, TM, gl_CooperativeMatrixLayoutColumnMajor);
381
+
382
+ barrier();
383
+ [[unroll]] for (uint col = 0; col < TN; col += storestride) {
384
+ const uint row_i = dc + cm_col * TN + col + store_c;
385
+ if (row_i >= _ne1) break;
386
+
387
+ const u16vec2 row_idx = row_ids[row_i - ic * BN];
388
+
389
+ if (dr + cm_row * TM + store_r < p.M) {
390
+ data_d[row_idx.y * p.batch_stride_d + row_idx.x * p.stride_d + dr + cm_row * TM + store_r] = D_TYPE(coopmat_stage[warp_i * TM * TN + (col + store_c) * TM + store_r]);
391
+ }
392
+ }
393
+ barrier();
394
+ }
395
+ }
396
+ #else
397
+ const bool is_aligned = p.stride_d % 4 == 0; // Assumption: D_TYPE == float
398
+
399
+ [[unroll]] for (uint cm_row = 0; cm_row < cms_per_row; cm_row++) {
400
+ [[unroll]] for (uint cm_col = 0; cm_col < cms_per_col; cm_col++) {
401
+ const bool is_in_bounds = dr + (cm_row + 1) * TM <= p.M && dc + (cm_col + 1) * TN <= p.N;
402
+
403
+ if (is_aligned && is_in_bounds) {
404
+ // Full coopMat is within bounds and stride_d is aligned with 16B
405
+ coopmat<D_TYPE, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator> cm_dtype = coopmat<D_TYPE, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator>(sums[cm_col * cms_per_row + cm_row]);
406
+ coopMatStore(cm_dtype, data_d, offsets + (dc + cm_col * TN) * p.stride_d + dr + cm_row * TM, p.stride_d, gl_CooperativeMatrixLayoutColumnMajor);
407
+ } else if (is_in_bounds) {
408
+ // Full coopMat is within bounds, but stride_d is not aligned
409
+ coopMatStore(sums[cm_col * cms_per_row + cm_row], coopmat_stage, warp_i * TM * TN, TM, gl_CooperativeMatrixLayoutColumnMajor);
410
+
411
+ controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease);
412
+ [[unroll]] for (uint col = 0; col < TN; col += storestride) {
413
+ data_d[offsets + (dc + cm_col * TN + col + store_c) * p.stride_d + dr + cm_row * TM + store_r] = D_TYPE(coopmat_stage[warp_i * TM * TN + (col + store_c) * TM + store_r]);
414
+ }
415
+ controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease);
416
+ } else if (dr + cm_row * TM < p.M && dc + cm_col * TN < p.N) {
417
+ // Partial coopMat is within bounds
418
+ coopMatStore(sums[cm_col * cms_per_row + cm_row], coopmat_stage, warp_i * TM * TN, TM, gl_CooperativeMatrixLayoutColumnMajor);
419
+
420
+ controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease);
421
+ [[unroll]] for (uint col = 0; col < TN; col += storestride) {
422
+ if (dr + cm_row * TM + store_r < p.M && dc + cm_col * TN + col + store_c < p.N) {
423
+ data_d[offsets + (dc + cm_col * TN + col + store_c) * p.stride_d + dr + cm_row * TM + store_r] = D_TYPE(coopmat_stage[warp_i * TM * TN + (col + store_c) * TM + store_r]);
424
+ }
425
+ }
426
+ controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease);
427
+ }
428
+ }
429
+ }
430
+ #endif // MUL_MAT_ID
431
+ #else
432
+ [[unroll]] for (uint wsic = 0; wsic < WNITER; wsic++) {
433
+ [[unroll]] for (uint wsir = 0; wsir < WMITER; wsir++) {
434
+
435
+ const uint dr_warp = dr + wsir * WSUBM + tiwr * TM;
436
+ const uint dc_warp = dc + wsic * WSUBN + tiwc * TN;
437
+ [[unroll]] for (uint cc = 0; cc < TN; cc++) {
438
+ #ifdef MUL_MAT_ID
439
+ const uint row_i = dc_warp + cc;
440
+ if (row_i >= _ne1) break;
441
+
442
+ const u16vec2 row_idx = row_ids[row_i - ic * BN];
443
+ #endif // MUL_MAT_ID
444
+ [[unroll]] for (uint cr = 0; cr < TM / 2; cr++) {
445
+ const uint sums_idx = (wsic * TN + cc) * WMITER * (TM / 2) + wsir * (TM / 2) + cr;
446
+ #ifdef MUL_MAT_ID
447
+ if (dr_warp + 2 * cr < p.M) {
448
+ data_d[row_idx.y * p.batch_stride_d + row_idx.x * p.stride_d + dr_warp + 2 * cr] = D_TYPE(sums[sums_idx].x);
449
+ }
450
+ if (dr_warp + 2 * cr + 1 < p.M) {
451
+ data_d[row_idx.y * p.batch_stride_d + row_idx.x * p.stride_d + dr_warp + 2 * cr + 1] = D_TYPE(sums[sums_idx].y);
452
+ }
453
+ #else
454
+ if (dr_warp + 2 * cr < p.M && dc_warp + cc < p.N) {
455
+ data_d[offsets + (dc_warp + cc) * p.stride_d + dr_warp + 2 * cr] = D_TYPE(sums[sums_idx].x);
456
+ }
457
+ if (dr_warp + 2 * cr + 1 < p.M && dc_warp + cc < p.N) {
458
+ data_d[offsets + (dc_warp + cc) * p.stride_d + dr_warp + 2 * cr + 1] = D_TYPE(sums[sums_idx].y);
459
+ }
460
+ #endif // MUL_MAT_ID
461
+ }
462
+ }
463
+ }
464
+ }
465
+ #endif // COOPMAT
466
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp ADDED
@@ -0,0 +1,658 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+
3
+ #extension GL_EXT_control_flow_attributes : enable
4
+ #extension GL_EXT_shader_16bit_storage : require
5
+
6
+ #extension GL_EXT_shader_explicit_arithmetic_types_float16 : require
7
+ #extension GL_EXT_shader_explicit_arithmetic_types_int8 : require
8
+ #extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
9
+ #extension GL_EXT_shader_explicit_arithmetic_types_int16 : require
10
+
11
+ #extension GL_KHR_memory_scope_semantics : enable
12
+ #extension GL_KHR_cooperative_matrix : enable
13
+ #extension GL_NV_cooperative_matrix2 : enable
14
+ #ifdef GGML_VULKAN_COOPMAT2_DECODE_VECTOR
15
+ #extension GL_NV_cooperative_matrix_decode_vector : enable
16
+ #endif
17
+ #extension GL_EXT_buffer_reference : enable
18
+ #extension GL_KHR_shader_subgroup_ballot : enable
19
+ #extension GL_KHR_shader_subgroup_vote : enable
20
+ #ifdef DATA_A_BF16
21
+ #extension GL_EXT_bfloat16 : enable
22
+ #endif
23
+
24
+ #include "types.glsl"
25
+ #include "utils.glsl"
26
+
27
+ layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
28
+
29
+ #define IS_MUL_MM2 1
30
+
31
+ layout (constant_id = 0) const uint BLOCK_SIZE = 256;
32
+ layout (constant_id = 1) const uint BM = 64;
33
+ layout (constant_id = 2) const uint BN = 64;
34
+ layout (constant_id = 3) const uint BK = 16; // Assumed to be 32 if working with a quant
35
+
36
+ layout (constant_id = 4) const bool enable_smaller_matrices = false;
37
+ const uint BNover2 = enable_smaller_matrices ? (BN / 2) : BN;
38
+ const uint BNover4 = enable_smaller_matrices ? (BN / 4) : BN;
39
+ layout (constant_id = 5) const uint ALIGNED = 0;
40
+
41
+ layout (push_constant) uniform parameter
42
+ {
43
+ uint M;
44
+ uint N;
45
+ uint K;
46
+ uint stride_a;
47
+ uint stride_b;
48
+ uint stride_d;
49
+
50
+ uint batch_stride_a;
51
+ uint batch_stride_b;
52
+ uint batch_stride_d;
53
+
54
+ #ifdef MUL_MAT_ID
55
+ uint nei0;
56
+ uint nei1;
57
+ uint nbi1;
58
+ uint ne11;
59
+ #else
60
+ uint base_work_group_z;
61
+ uint num_batches;
62
+ uint k_split;
63
+ uint ne02;
64
+ uint ne12;
65
+ uint broadcast2;
66
+ uint broadcast3;
67
+ #endif
68
+ // N dimension for the B matrix can be >= p.N
69
+ uint padded_N;
70
+ } p;
71
+
72
+
73
+ layout (binding = 0) readonly buffer A {A_TYPE data_a[];};
74
+ layout (binding = 1) readonly buffer B {B_TYPE data_b[];};
75
+ layout (binding = 2) writeonly buffer D {D_TYPE data_d[];};
76
+ #if defined(MUL_MAT_ID) && defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR)
77
+ layout (binding = 1) readonly buffer B4 {B_TYPEV4 data_b_v4[];};
78
+ #endif
79
+
80
+ #if QUANT_K > 1
81
+ #include "dequant_funcs_cm2.glsl"
82
+ #if defined(dequantFuncA_v) && defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR)
83
+ #define DECODEFUNCA , dequantFuncA, dequantFuncA_v
84
+ #else
85
+ #define DECODEFUNCA , dequantFuncA
86
+ #endif
87
+ #else
88
+ #define DECODEFUNCA
89
+ #endif
90
+
91
+ #if !defined(fetch_scales)
92
+ #define fetch_scales(a, b, c, d, e, f)
93
+ #endif
94
+ #if !defined(store_scales)
95
+ #define store_scales(a)
96
+ #endif
97
+
98
+ #if defined(DATA_A_BF16)
99
+ #define MAT_TYPE bfloat16_t
100
+ #else
101
+ #define MAT_TYPE FLOAT_TYPE
102
+ #endif
103
+
104
+ #ifdef MUL_MAT_ID
105
+ layout (binding = 3) readonly buffer IDS {int data_ids[];};
106
+ layout (binding = 4) readonly buffer Counts {int data_expert_count[];};
107
+
108
+ shared u16vec4 row_ids[BN];
109
+
110
+ layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufB {
111
+ B_TYPE b[];
112
+ };
113
+
114
+ uint _ne1;
115
+ layout (constant_id = 6) const uint subgroup_size = 32;
116
+ shared uvec4 ballots_sh[BLOCK_SIZE / subgroup_size];
117
+
118
+ B_TYPE decodeFuncB(const in decodeBufB bl, const in uint blockCoords[2], const in uint coordInBlock[2])
119
+ {
120
+ const uint row_i = blockCoords[0];
121
+
122
+ const u16vec4 row_idx = row_ids[row_i];
123
+ #if defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR)
124
+ // The decode-vector path gives B a K-dimension tensor-layout block size of BK.
125
+ const uint k = blockCoords[1] * BK + coordInBlock[1];
126
+ #else
127
+ const uint k = blockCoords[1];
128
+ #endif
129
+ B_TYPE ret = data_b[row_idx.y * p.batch_stride_b + row_idx.x * p.stride_b + k];
130
+
131
+ return ret;
132
+ }
133
+
134
+ #if defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR)
135
+ B_TYPEV4 decodeFuncB_v(const in decodeBufB bl, const in uint blockCoords[2], const in uint coordInBlock[2])
136
+ {
137
+ const uint row_i = blockCoords[0];
138
+
139
+ const u16vec4 row_idx = row_ids[row_i];
140
+ const uint k = blockCoords[1] * BK + coordInBlock[1];
141
+ const uint base = row_idx.y * p.batch_stride_b + row_idx.x * p.stride_b + k;
142
+
143
+ return data_b_v4[base >> 2];
144
+ }
145
+ #define DECODEFUNCB , decodeFuncB, decodeFuncB_v
146
+ #else
147
+ #define DECODEFUNCB , decodeFuncB
148
+ #endif
149
+
150
+ D_TYPE perElemOpD(const in uint32_t r, const in uint32_t c, const in D_TYPE elem, const in uint32_t ir, const in uint32_t ic)
151
+ {
152
+ uint dr = ir * BM + r;
153
+ uint dc = ic * BN + c;
154
+
155
+ if (dr < p.M && dc < _ne1) {
156
+ uint row_i = c;
157
+ const u16vec4 row_idx = row_ids[row_i];
158
+ data_d[row_idx.y * p.batch_stride_d + row_idx.z * p.stride_d + dr] = elem;
159
+ }
160
+ return elem;
161
+ }
162
+
163
+ void load_row_ids(uint expert_idx, bool nei0_is_pow2, uint ic) {
164
+ _ne1 = 0;
165
+ uint num_elements = p.nei1 * p.nei0;
166
+ uint nei0shift = findLSB(p.nei0);
167
+
168
+ uint ids[16];
169
+ uint iter = 0;
170
+
171
+ uint expert_count = data_expert_count[expert_idx];
172
+
173
+ for (uint j = 0; j < num_elements; j += BLOCK_SIZE) {
174
+ // prefetch up to 16 elements
175
+ if (iter == 0) {
176
+ [[unroll]] for (uint k = 0; k < 16; ++k) {
177
+ uint i = j + gl_LocalInvocationIndex + k*BLOCK_SIZE;
178
+ bool in_range = i < num_elements;
179
+ uint ii1;
180
+ if (nei0_is_pow2) {
181
+ ii1 = i >> nei0shift;
182
+ } else {
183
+ ii1 = i / p.nei0;
184
+ }
185
+ uint ii0 = i - ii1 * p.nei0;
186
+ ids[k] = in_range ? data_ids[ii1*p.nbi1 + ii0] : 0;
187
+ }
188
+ }
189
+ uint i = j + gl_LocalInvocationIndex;
190
+ bool in_range = i < num_elements;
191
+ uint ii1;
192
+ if (nei0_is_pow2) {
193
+ ii1 = i >> nei0shift;
194
+ } else {
195
+ ii1 = i / p.nei0;
196
+ }
197
+ uint ii0 = i - ii1 * p.nei0;
198
+ uint id = ids[iter++];
199
+ uvec4 ballot = subgroupBallot(in_range && id == expert_idx);
200
+
201
+ if (gl_SubgroupInvocationID == 0) {
202
+ ballots_sh[gl_SubgroupID] = ballot;
203
+ }
204
+ barrier();
205
+
206
+ uint subgroup_base = 0;
207
+ uint total = 0;
208
+ for (uint k = 0; k < gl_NumSubgroups; ++k) {
209
+ if (k == gl_SubgroupID) {
210
+ subgroup_base = total;
211
+ }
212
+ total += subgroupBallotBitCount(ballots_sh[k]);
213
+ }
214
+ barrier();
215
+
216
+ uint idx = subgroup_base + subgroupBallotExclusiveBitCount(ballot);
217
+ if (in_range && id == expert_idx && _ne1 + idx >= ic * BN && _ne1 + idx < (ic + 1) * BN) {
218
+ row_ids[_ne1 + idx - ic * BN] = u16vec4(fastmod(ii0, p.ne11), ii1, ii0, 0);
219
+ }
220
+ _ne1 += total;
221
+ iter &= 15;
222
+ if (_ne1 >= (ic + 1) * BN || _ne1 == expert_count) {
223
+ break;
224
+ }
225
+ }
226
+ barrier();
227
+ }
228
+ #endif
229
+
230
+ void main() {
231
+ const uint tid = gl_LocalInvocationIndex;
232
+ const uint ic = gl_WorkGroupID.y;
233
+
234
+ #ifdef MUL_MAT_ID
235
+ const uint expert_idx = gl_WorkGroupID.z;
236
+ if (ic * BN >= data_expert_count[expert_idx]) {
237
+ return;
238
+ }
239
+ // initialize to row 0 so we don't need to bounds check
240
+ if (tid < BN) {
241
+ row_ids[tid] = u16vec4(0);
242
+ }
243
+ #if !defined(NEEDS_INIT_IQ_SHMEM)
244
+ barrier();
245
+ #endif
246
+ #endif
247
+
248
+ #ifdef NEEDS_INIT_IQ_SHMEM
249
+ init_iq_shmem(gl_WorkGroupSize);
250
+ #endif
251
+
252
+ #ifndef MUL_MAT_ID
253
+ const uint batch_idx = gl_WorkGroupID.z + p.base_work_group_z;
254
+
255
+ const uint i13 = batch_idx / p.ne12;
256
+ const uint i12 = batch_idx % p.ne12;
257
+
258
+ const uint i03 = i13 / p.broadcast3;
259
+ const uint i02 = i12 / p.broadcast2;
260
+
261
+ const uint batch_idx_a = i03 * p.ne02 + i02;
262
+ #endif
263
+
264
+ const uint blocks_m = (p.M + BM - 1) / BM;
265
+ const uint ir = gl_WorkGroupID.x % blocks_m;
266
+ const uint ik = gl_WorkGroupID.x / blocks_m;
267
+
268
+ #ifdef MUL_MAT_ID
269
+ if (bitCount(p.nei0) == 1) {
270
+ load_row_ids(expert_idx, true, ic);
271
+ } else {
272
+ load_row_ids(expert_idx, false, ic);
273
+ }
274
+
275
+ // Workgroup has no work
276
+ if (ic * BN >= _ne1) return;
277
+ #endif
278
+
279
+ #ifdef MUL_MAT_ID
280
+ uint start_k = 0;
281
+ const uint end_k = p.K;
282
+ #else
283
+ uint start_k = ik * p.k_split;
284
+ const uint end_k = min(p.K, (ik + 1) * p.k_split);
285
+ #endif
286
+
287
+ #ifdef MUL_MAT_ID
288
+ uint pos_a = expert_idx * (p.batch_stride_a / QUANT_K);
289
+ uint pos_b = 0;
290
+ #else
291
+ uint pos_a = batch_idx_a * (p.batch_stride_a / QUANT_K);
292
+ uint pos_b = batch_idx * p.batch_stride_b;
293
+ uint pos_d = batch_idx * p.batch_stride_d + ik * p.batch_stride_d * p.num_batches;
294
+ #endif
295
+
296
+ uint stride_a = p.stride_a / QUANT_K;
297
+ uint stride_b = p.stride_b;
298
+
299
+ // Hint to the compiler that values are aligned (want 16B alignment).
300
+ // Quants are always block-aligned, no alignment needed.
301
+ if (ALIGNED != 0) {
302
+ #if QUANT_K == 1
303
+ stride_a &= ~7;
304
+ #endif
305
+ stride_b &= ~7;
306
+ }
307
+
308
+ // Create layouts for both clamped and unclamped accesses
309
+ tensorLayoutNV<2> tensorLayoutA = createTensorLayoutNV(2);
310
+ tensorLayoutNV<2, gl_CooperativeMatrixClampModeConstantNV> tensorLayoutAClamp = createTensorLayoutNV(2, gl_CooperativeMatrixClampModeConstantNV);
311
+ tensorLayoutNV<2> tensorLayoutB = createTensorLayoutNV(2);
312
+ tensorLayoutNV<2, gl_CooperativeMatrixClampModeConstantNV> tensorLayoutBClamp = createTensorLayoutNV(2, gl_CooperativeMatrixClampModeConstantNV);
313
+ tensorLayoutNV<2, gl_CooperativeMatrixClampModeConstantNV> tensorLayoutD = createTensorLayoutNV(2, gl_CooperativeMatrixClampModeConstantNV);
314
+
315
+ #if QUANT_K > 1
316
+ tensorLayoutA = setTensorLayoutBlockSizeNV(tensorLayoutA, 1, QUANT_K);
317
+ tensorLayoutAClamp = setTensorLayoutBlockSizeNV(tensorLayoutAClamp, 1, QUANT_K);
318
+ #endif
319
+ #if defined(MUL_MAT_ID) && defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR)
320
+ tensorLayoutB = setTensorLayoutBlockSizeNV(tensorLayoutB, 1, BK);
321
+ #endif
322
+
323
+ // Use end_k rather than p.K as the dimension because that's what
324
+ // we need to bound check against when using split_k.
325
+ // Bounds check B against padded_N, but bounds check D against N.
326
+ tensorLayoutA = setTensorLayoutDimensionNV(tensorLayoutA, p.M, end_k);
327
+ tensorLayoutB = setTensorLayoutDimensionNV(tensorLayoutB, p.padded_N, end_k);
328
+ tensorLayoutD = setTensorLayoutDimensionNV(tensorLayoutD, p.N, p.M);
329
+ tensorLayoutAClamp = setTensorLayoutDimensionNV(tensorLayoutAClamp, p.M, end_k);
330
+ tensorLayoutBClamp = setTensorLayoutDimensionNV(tensorLayoutBClamp, p.padded_N, end_k);
331
+
332
+ tensorLayoutD = setTensorLayoutStrideNV(tensorLayoutD, p.stride_d, 1);
333
+
334
+ tensorViewNV<2, false, 1, 0> tensorViewTranspose = createTensorViewNV(2, false, 1, 0);
335
+
336
+ #if !defined(MUL_MAT_ID)
337
+
338
+ const uint START_ALIGN_K = 256;
339
+ // For Qi_K (block size 256), unroll whole 256 element tiles.
340
+ // For legacy quants (block size 32), unroll 8x.
341
+ const uint UNROLL_K = (QUANT_K == 256) ? 256 : (BK * 8);
342
+ const uint unroll_count = UNROLL_K / BK;
343
+
344
+ // Detect a fast path where all loads are entirely in bounds and no clamping is required
345
+ if ((ir + 1) * BM <= p.M && (ic + 1) * BN <= p.padded_N && (start_k % START_ALIGN_K) == 0 && (end_k % BK) == 0 &&
346
+ #if QUANT_K == 1
347
+ (stride_a % 8) == 0 &&
348
+ #endif
349
+ (stride_b % 8) == 0) {
350
+ // Hint to the compiler that values are aligned (want 16B alignment)
351
+ start_k &= ~(START_ALIGN_K-1);
352
+ stride_b &= ~7;
353
+ #if QUANT_K == 1
354
+ stride_a &= ~7;
355
+ #endif
356
+
357
+ tensorLayoutA = setTensorLayoutStrideNV(tensorLayoutA, stride_a, 1);
358
+ tensorLayoutB = setTensorLayoutStrideNV(tensorLayoutB, stride_b, 1);
359
+
360
+ uint k_iters = (end_k - start_k) / UNROLL_K;
361
+ uint block_k = start_k;
362
+
363
+ // fetch scale values for a tile of quants. These will be copied into shared memory.
364
+ // The fetches and stores are pipelined to hide the latency.
365
+ fetch_scales(ir * BM, pos_a, stride_a, start_k, tid, true);
366
+
367
+ if (enable_smaller_matrices && ic * BN + BNover4 >= p.N) {
368
+ coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BNover4, gl_MatrixUseAccumulator> sum = coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BNover4, gl_MatrixUseAccumulator>(0.0);
369
+ for (uint i = 0; i < k_iters; ++i) {
370
+
371
+ store_scales(tid);
372
+ if (block_k + UNROLL_K < end_k) {
373
+ fetch_scales(ir * BM, pos_a, stride_a, block_k + UNROLL_K, tid, true);
374
+ }
375
+
376
+ // Manually partial unroll
377
+ [[unroll]] for (uint j = 0; j < unroll_count; ++j) {
378
+ coopmat<MAT_TYPE, gl_ScopeWorkgroup, BM, BK, gl_MatrixUseA> mat_a;
379
+ coopmat<MAT_TYPE, gl_ScopeWorkgroup, BK, BNover4, gl_MatrixUseB> mat_b;
380
+
381
+ coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA);
382
+ coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, ic * BN, BNover4, block_k, BK), tensorViewTranspose);
383
+
384
+ sum = coopMatMulAdd(mat_a, mat_b, sum);
385
+ block_k += BK;
386
+ }
387
+ }
388
+ // Do any remaining iterations that were not unrolled
389
+ if (block_k < end_k) {
390
+ store_scales(tid);
391
+ }
392
+ while (block_k < end_k) {
393
+ coopmat<MAT_TYPE, gl_ScopeWorkgroup, BM, BK, gl_MatrixUseA> mat_a;
394
+ coopmat<MAT_TYPE, gl_ScopeWorkgroup, BK, BNover4, gl_MatrixUseB> mat_b;
395
+
396
+ coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA);
397
+ coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, ic * BN, BNover4, block_k, BK), tensorViewTranspose);
398
+
399
+ sum = coopMatMulAdd(mat_a, mat_b, sum);
400
+ block_k += BK;
401
+ }
402
+ #if defined(ACC_TYPE_MAX)
403
+ [[unroll]] for (uint i = 0; i < sum.length(); ++i) { sum[i] = clamp(sum[i], -ACC_TYPE_MAX, ACC_TYPE_MAX); }
404
+ #endif
405
+
406
+ coopmat<D_TYPE, gl_ScopeWorkgroup, BM, BNover4, gl_MatrixUseAccumulator> mat_d = coopmat<D_TYPE, gl_ScopeWorkgroup, BM, BNover4, gl_MatrixUseAccumulator>(sum);
407
+
408
+ coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BNover4, ir * BM, BM), tensorViewTranspose);
409
+ return;
410
+ } else if (enable_smaller_matrices && ic * BN + BNover2 >= p.N) {
411
+ coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BNover2, gl_MatrixUseAccumulator> sum = coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BNover2, gl_MatrixUseAccumulator>(0.0);
412
+ for (uint i = 0; i < k_iters; ++i) {
413
+
414
+ store_scales(tid);
415
+ if (block_k + UNROLL_K < end_k) {
416
+ fetch_scales(ir * BM, pos_a, stride_a, block_k + UNROLL_K, tid, true);
417
+ }
418
+
419
+ // Manually partial unroll
420
+ [[unroll]] for (uint j = 0; j < unroll_count; ++j) {
421
+ coopmat<MAT_TYPE, gl_ScopeWorkgroup, BM, BK, gl_MatrixUseA> mat_a;
422
+ coopmat<MAT_TYPE, gl_ScopeWorkgroup, BK, BNover2, gl_MatrixUseB> mat_b;
423
+
424
+ coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA);
425
+ coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, ic * BN, BNover2, block_k, BK), tensorViewTranspose);
426
+
427
+ sum = coopMatMulAdd(mat_a, mat_b, sum);
428
+ block_k += BK;
429
+ }
430
+ }
431
+ // Do any remaining iterations that were not unrolled
432
+ if (block_k < end_k) {
433
+ store_scales(tid);
434
+ }
435
+ while (block_k < end_k) {
436
+ coopmat<MAT_TYPE, gl_ScopeWorkgroup, BM, BK, gl_MatrixUseA> mat_a;
437
+ coopmat<MAT_TYPE, gl_ScopeWorkgroup, BK, BNover2, gl_MatrixUseB> mat_b;
438
+
439
+ coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA);
440
+ coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, ic * BN, BNover2, block_k, BK), tensorViewTranspose);
441
+
442
+ sum = coopMatMulAdd(mat_a, mat_b, sum);
443
+ block_k += BK;
444
+ }
445
+ #if defined(ACC_TYPE_MAX)
446
+ [[unroll]] for (uint i = 0; i < sum.length(); ++i) { sum[i] = clamp(sum[i], -ACC_TYPE_MAX, ACC_TYPE_MAX); }
447
+ #endif
448
+
449
+ coopmat<D_TYPE, gl_ScopeWorkgroup, BM, BNover2, gl_MatrixUseAccumulator> mat_d = coopmat<D_TYPE, gl_ScopeWorkgroup, BM, BNover2, gl_MatrixUseAccumulator>(sum);
450
+
451
+ coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BNover2, ir * BM, BM), tensorViewTranspose);
452
+ return;
453
+ } else {
454
+ coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BN, gl_MatrixUseAccumulator> sum = coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BN, gl_MatrixUseAccumulator>(0.0);
455
+
456
+ for (uint i = 0; i < k_iters; ++i) {
457
+
458
+ store_scales(tid);
459
+ if (block_k + UNROLL_K < end_k) {
460
+ fetch_scales(ir * BM, pos_a, stride_a, block_k + UNROLL_K, tid, true);
461
+ }
462
+
463
+ // Manually partial unroll
464
+ [[unroll]] for (uint j = 0; j < unroll_count; ++j) {
465
+ coopmat<MAT_TYPE, gl_ScopeWorkgroup, BM, BK, gl_MatrixUseA> mat_a;
466
+ coopmat<MAT_TYPE, gl_ScopeWorkgroup, BK, BN, gl_MatrixUseB> mat_b;
467
+
468
+ coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA);
469
+ coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, ic * BN, BN, block_k, BK), tensorViewTranspose);
470
+
471
+ sum = coopMatMulAdd(mat_a, mat_b, sum);
472
+ block_k += BK;
473
+ }
474
+ }
475
+ // Do any remaining iterations that were not unrolled
476
+ if (block_k < end_k) {
477
+ store_scales(tid);
478
+ }
479
+ while (block_k < end_k) {
480
+ coopmat<MAT_TYPE, gl_ScopeWorkgroup, BM, BK, gl_MatrixUseA> mat_a;
481
+ coopmat<MAT_TYPE, gl_ScopeWorkgroup, BK, BN, gl_MatrixUseB> mat_b;
482
+
483
+ coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA);
484
+ coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, ic * BN, BN, block_k, BK), tensorViewTranspose);
485
+
486
+ sum = coopMatMulAdd(mat_a, mat_b, sum);
487
+ block_k += BK;
488
+ }
489
+ #if defined(ACC_TYPE_MAX)
490
+ [[unroll]] for (uint i = 0; i < sum.length(); ++i) { sum[i] = clamp(sum[i], -ACC_TYPE_MAX, ACC_TYPE_MAX); }
491
+ #endif
492
+
493
+ coopmat<D_TYPE, gl_ScopeWorkgroup, BM, BN, gl_MatrixUseAccumulator> mat_d = coopmat<D_TYPE, gl_ScopeWorkgroup, BM, BN, gl_MatrixUseAccumulator>(sum);
494
+
495
+ coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BN, ir * BM, BM), tensorViewTranspose);
496
+ return;
497
+ }
498
+ } else
499
+ #endif // !defined(MUL_MAT_ID)
500
+ {
501
+ tensorLayoutA = setTensorLayoutStrideNV(tensorLayoutA, stride_a, 1);
502
+
503
+ tensorLayoutAClamp = setTensorLayoutStrideNV(tensorLayoutAClamp, stride_a, 1);
504
+
505
+ tensorLayoutB = setTensorLayoutStrideNV(tensorLayoutB, stride_b, 1);
506
+
507
+ tensorLayoutBClamp = setTensorLayoutStrideNV(tensorLayoutBClamp, stride_b, 1);
508
+
509
+ uint k_iters = (end_k - start_k + BK - 1) / BK;
510
+
511
+ fetch_scales(ir * BM, pos_a, stride_a, start_k, tid, false);
512
+ store_scales(tid);
513
+
514
+ #ifdef MUL_MAT_ID
515
+ if (enable_smaller_matrices && ic * BN + BNover4 >= _ne1) {
516
+ coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BNover4, gl_MatrixUseAccumulator> sum;
517
+ sum = coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BNover4, gl_MatrixUseAccumulator>(0.0);
518
+
519
+ [[dont_unroll]]
520
+ for (uint block_k = start_k, i = 0; i < k_iters; block_k += BK, ++i) {
521
+
522
+ if ((block_k % QUANT_K) == 0) {
523
+ store_scales(tid);
524
+ }
525
+ if (block_k + BK < end_k && ((block_k + BK) % QUANT_K) == 0) {
526
+ fetch_scales(ir * BM, pos_a, stride_a, block_k + BK, tid, false);
527
+ }
528
+
529
+ if ((ir + 1) * BM <= p.M && block_k + BK <= end_k) {
530
+ coopmat<MAT_TYPE, gl_ScopeWorkgroup, BM, BK, gl_MatrixUseA> mat_a;
531
+ coopmat<MAT_TYPE, gl_ScopeWorkgroup, BK, BNover4, gl_MatrixUseB> mat_b;
532
+
533
+ coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA);
534
+ coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, 0, BNover4, block_k, BK), tensorViewTranspose DECODEFUNCB);
535
+
536
+ sum = coopMatMulAdd(mat_a, mat_b, sum);
537
+ } else {
538
+ coopmat<MAT_TYPE, gl_ScopeWorkgroup, BM, BK, gl_MatrixUseA> mat_a;
539
+ coopmat<MAT_TYPE, gl_ScopeWorkgroup, BK, BNover4, gl_MatrixUseB> mat_b;
540
+
541
+ coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutAClamp, ir * BM, BM, block_k, BK) DECODEFUNCA);
542
+ coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, 0, BNover4, block_k, BK), tensorViewTranspose DECODEFUNCB);
543
+
544
+ sum = coopMatMulAdd(mat_a, mat_b, sum);
545
+ }
546
+ }
547
+ #if defined(ACC_TYPE_MAX)
548
+ [[unroll]] for (uint i = 0; i < sum.length(); ++i) { sum[i] = clamp(sum[i], -ACC_TYPE_MAX, ACC_TYPE_MAX); }
549
+ #endif
550
+
551
+ // Convert from ACC_TYPE to D_TYPE
552
+ coopmat<D_TYPE, gl_ScopeWorkgroup, BM, BNover4, gl_MatrixUseAccumulator> mat_d;
553
+ mat_d = coopmat<D_TYPE, gl_ScopeWorkgroup, BM, BNover4, gl_MatrixUseAccumulator>(sum);
554
+
555
+ // Call callback to store each element, remapping row through shared memory
556
+ coopMatPerElementNV(mat_d, mat_d, perElemOpD, ir, ic);
557
+ return;
558
+ }
559
+ if (enable_smaller_matrices && ic * BN + BNover2 >= _ne1) {
560
+ coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BNover2, gl_MatrixUseAccumulator> sum;
561
+ sum = coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BNover2, gl_MatrixUseAccumulator>(0.0);
562
+
563
+ [[dont_unroll]]
564
+ for (uint block_k = start_k, i = 0; i < k_iters; block_k += BK, ++i) {
565
+
566
+ if ((block_k % QUANT_K) == 0) {
567
+ store_scales(tid);
568
+ }
569
+ if (block_k + BK < end_k && ((block_k + BK) % QUANT_K) == 0) {
570
+ fetch_scales(ir * BM, pos_a, stride_a, block_k + BK, tid, false);
571
+ }
572
+
573
+ if ((ir + 1) * BM <= p.M && block_k + BK <= end_k) {
574
+ coopmat<MAT_TYPE, gl_ScopeWorkgroup, BM, BK, gl_MatrixUseA> mat_a;
575
+ coopmat<MAT_TYPE, gl_ScopeWorkgroup, BK, BNover2, gl_MatrixUseB> mat_b;
576
+
577
+ coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA);
578
+ coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, 0, BNover2, block_k, BK), tensorViewTranspose DECODEFUNCB);
579
+
580
+ sum = coopMatMulAdd(mat_a, mat_b, sum);
581
+ } else {
582
+ coopmat<MAT_TYPE, gl_ScopeWorkgroup, BM, BK, gl_MatrixUseA> mat_a;
583
+ coopmat<MAT_TYPE, gl_ScopeWorkgroup, BK, BNover2, gl_MatrixUseB> mat_b;
584
+
585
+ coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutAClamp, ir * BM, BM, block_k, BK) DECODEFUNCA);
586
+ coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, 0, BNover2, block_k, BK), tensorViewTranspose DECODEFUNCB);
587
+
588
+ sum = coopMatMulAdd(mat_a, mat_b, sum);
589
+ }
590
+ }
591
+ #if defined(ACC_TYPE_MAX)
592
+ [[unroll]] for (uint i = 0; i < sum.length(); ++i) { sum[i] = clamp(sum[i], -ACC_TYPE_MAX, ACC_TYPE_MAX); }
593
+ #endif
594
+
595
+ // Convert from ACC_TYPE to D_TYPE
596
+ coopmat<D_TYPE, gl_ScopeWorkgroup, BM, BNover2, gl_MatrixUseAccumulator> mat_d;
597
+ mat_d = coopmat<D_TYPE, gl_ScopeWorkgroup, BM, BNover2, gl_MatrixUseAccumulator>(sum);
598
+
599
+ // Call callback to store each element, remapping row through shared memory
600
+ coopMatPerElementNV(mat_d, mat_d, perElemOpD, ir, ic);
601
+ return;
602
+ }
603
+ #endif
604
+ coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BN, gl_MatrixUseAccumulator> sum;
605
+ sum = coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BN, gl_MatrixUseAccumulator>(0.0);
606
+
607
+ [[dont_unroll]]
608
+ for (uint block_k = start_k, i = 0; i < k_iters; block_k += BK, ++i) {
609
+
610
+ if ((block_k % QUANT_K) == 0) {
611
+ store_scales(tid);
612
+ }
613
+ if (block_k + BK < end_k && ((block_k + BK) % QUANT_K) == 0) {
614
+ fetch_scales(ir * BM, pos_a, stride_a, block_k + BK, tid, false);
615
+ }
616
+
617
+ if ((ir + 1) * BM <= p.M && block_k + BK <= end_k) {
618
+ coopmat<MAT_TYPE, gl_ScopeWorkgroup, BM, BK, gl_MatrixUseA> mat_a;
619
+ coopmat<MAT_TYPE, gl_ScopeWorkgroup, BK, BN, gl_MatrixUseB> mat_b;
620
+
621
+ coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA);
622
+ #ifdef MUL_MAT_ID
623
+ coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, 0, BN, block_k, BK), tensorViewTranspose DECODEFUNCB);
624
+ #else
625
+ coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutBClamp, ic * BN, BN, block_k, BK), tensorViewTranspose);
626
+ #endif
627
+
628
+ sum = coopMatMulAdd(mat_a, mat_b, sum);
629
+ } else {
630
+ coopmat<MAT_TYPE, gl_ScopeWorkgroup, BM, BK, gl_MatrixUseA> mat_a;
631
+ coopmat<MAT_TYPE, gl_ScopeWorkgroup, BK, BN, gl_MatrixUseB> mat_b;
632
+
633
+ coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutAClamp, ir * BM, BM, block_k, BK) DECODEFUNCA);
634
+ #ifdef MUL_MAT_ID
635
+ coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, 0, BN, block_k, BK), tensorViewTranspose DECODEFUNCB);
636
+ #else
637
+ coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutBClamp, ic * BN, BN, block_k, BK), tensorViewTranspose);
638
+ #endif
639
+
640
+ sum = coopMatMulAdd(mat_a, mat_b, sum);
641
+ }
642
+ }
643
+ #if defined(ACC_TYPE_MAX)
644
+ [[unroll]] for (uint i = 0; i < sum.length(); ++i) { sum[i] = clamp(sum[i], -ACC_TYPE_MAX, ACC_TYPE_MAX); }
645
+ #endif
646
+
647
+ // Convert from ACC_TYPE to D_TYPE
648
+ coopmat<D_TYPE, gl_ScopeWorkgroup, BM, BN, gl_MatrixUseAccumulator> mat_d;
649
+ mat_d = coopmat<D_TYPE, gl_ScopeWorkgroup, BM, BN, gl_MatrixUseAccumulator>(sum);
650
+
651
+ #ifdef MUL_MAT_ID
652
+ // Call callback to store each element, remapping row through shared memory
653
+ coopMatPerElementNV(mat_d, mat_d, perElemOpD, ir, ic);
654
+ #else
655
+ coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BN, ir * BM, BM), tensorViewTranspose);
656
+ #endif
657
+ }
658
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl ADDED
@@ -0,0 +1,617 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uint idx_m, const uint block, const uint end_k) {
2
+ #if defined(DATA_A_F32) || defined(DATA_A_F16)
3
+ #if LOAD_VEC_A == 8
4
+ if (ALIGNED != 0) {
5
+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
6
+ const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
7
+ FLOAT_TYPEV8 aa = FLOAT_TYPEV8(data_a[idx]);
8
+ buf_a[buf_idx ] = aa[0].xy;
9
+ buf_a[buf_idx + 1] = aa[0].zw;
10
+ buf_a[buf_idx + 2] = aa[1].xy;
11
+ buf_a[buf_idx + 3] = aa[1].zw;
12
+ return;
13
+ }
14
+ #elif LOAD_VEC_A == 4
15
+ if (ALIGNED != 0) {
16
+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
17
+ const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
18
+ FLOAT_TYPEV4 aa = FLOAT_TYPEV4(data_a[idx]);
19
+ buf_a[buf_idx ] = aa.xy;
20
+ buf_a[buf_idx + 1] = aa.zw;
21
+ return;
22
+ }
23
+ #endif
24
+ const uint idx = pos_a + col * p.stride_a + row * 2;
25
+ const uint buf_idx = col * SHMEM_STRIDE + row;
26
+ if (idx_m < p.M && block + row * 2 + 1 < end_k) {
27
+ buf_a[buf_idx] = FLOAT_TYPEV2(data_a_scalar[idx],
28
+ data_a_scalar[idx + 1]);
29
+ } else if (idx_m < p.M && block + row * 2 < end_k) {
30
+ buf_a[buf_idx] = FLOAT_TYPEV2(data_a_scalar[idx], 0.0f);
31
+ } else {
32
+ buf_a[buf_idx] = FLOAT_TYPEV2(0.0f);
33
+ }
34
+ #elif defined(DATA_A_BF16)
35
+ #if LOAD_VEC_A == 4
36
+ if (ALIGNED != 0) {
37
+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
38
+ const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
39
+ FLOAT_TYPEV4 aa = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_a[idx]));
40
+ buf_a[buf_idx ] = aa.xy;
41
+ buf_a[buf_idx + 1] = aa.zw;
42
+ return;
43
+ }
44
+ #endif
45
+ const uint idx = pos_a + col * p.stride_a + row * 2;
46
+ const uint buf_idx = col * SHMEM_STRIDE + row;
47
+ if (idx_m < p.M && block + row * 2 + 1 < end_k) {
48
+ buf_a[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_a_scalar[idx]),
49
+ TO_FLOAT_TYPE(data_a_scalar[idx + 1]));
50
+ } else if (idx_m < p.M && block + row * 2 < end_k) {
51
+ buf_a[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_a_scalar[idx]), 0.0f);
52
+ } else {
53
+ buf_a[buf_idx] = FLOAT_TYPEV2(0.0f);
54
+ }
55
+ #elif defined(DATA_A_Q4_0)
56
+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
57
+ const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 4;
58
+
59
+ const uint ib = idx / 4;
60
+ const uint iqs = idx & 0x03;
61
+
62
+ const float d = float(data_a_packed16[ib].d);
63
+ const uint vui = uint(data_a_packed16[ib].qs[2*iqs]) | (uint(data_a_packed16[ib].qs[2*iqs + 1]) << 16);
64
+ const vec4 v0 = (vec4(unpack8(vui & 0x0F0F0F0F)) - 8.0f) * d;
65
+ const vec4 v1 = (vec4(unpack8((vui >> 4) & 0x0F0F0F0F)) - 8.0f) * d;
66
+
67
+ buf_a[buf_idx ] = FLOAT_TYPEV2(v0.xy);
68
+ buf_a[buf_idx + 1] = FLOAT_TYPEV2(v0.zw);
69
+ buf_a[buf_idx + 8] = FLOAT_TYPEV2(v1.xy);
70
+ buf_a[buf_idx + 9] = FLOAT_TYPEV2(v1.zw);
71
+ #elif defined(DATA_A_Q4_1)
72
+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
73
+ const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 4;
74
+
75
+ const uint ib = idx / 4;
76
+ const uint iqs = idx & 0x03;
77
+
78
+ const vec2 dm = vec2(data_a_packed32[ib].dm);
79
+ const uint vui = data_a_packed32[ib].qs[iqs];
80
+ const vec4 v0 = vec4(unpack8(vui & 0x0F0F0F0F)) * dm.x + dm.y;
81
+ const vec4 v1 = vec4(unpack8((vui >> 4) & 0x0F0F0F0F)) * dm.x + dm.y;
82
+
83
+ buf_a[buf_idx ] = FLOAT_TYPEV2(v0.xy);
84
+ buf_a[buf_idx + 1 ] = FLOAT_TYPEV2(v0.zw);
85
+ buf_a[buf_idx + 8 ] = FLOAT_TYPEV2(v1.xy);
86
+ buf_a[buf_idx + 9 ] = FLOAT_TYPEV2(v1.zw);
87
+ #elif defined(DATA_A_Q5_0)
88
+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
89
+ const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 4;
90
+
91
+ const uint ib = idx / 8;
92
+ const uint iqs = idx & 0x07;
93
+
94
+ const float d = float(data_a_packed16[ib].d);
95
+ const uint uint_qh = uint(data_a_packed16[ib].qh[1]) << 16 | uint(data_a_packed16[ib].qh[0]);
96
+ const ivec2 qh0 = ivec2(((uint_qh >> 2*iqs) << 4) & 0x10, (uint_qh >> (2*iqs + 12)) & 0x10);
97
+ const ivec2 qh1 = ivec2(((uint_qh >> (2*iqs + 1)) << 4) & 0x10, (uint_qh >> (2*iqs + 13)) & 0x10);
98
+
99
+ const uint vui = uint(data_a_packed16[ib].qs[iqs]);
100
+ const vec4 v = (vec4((vui & 0xF) | qh0.x, ((vui >> 4) & 0xF) | qh0.y, ((vui >> 8) & 0xF) | qh1.x, (vui >> 12) | qh1.y) - 16.0f) * d;
101
+
102
+ buf_a[buf_idx ] = FLOAT_TYPEV2(v.xz);
103
+ buf_a[buf_idx + 8] = FLOAT_TYPEV2(v.yw);
104
+ #elif defined(DATA_A_Q5_1)
105
+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
106
+ const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 4;
107
+
108
+ const uint ib = idx / 4;
109
+ const uint iqs = idx & 0x03;
110
+
111
+ const vec2 dm = vec2(data_a_packed32[ib].dm);
112
+ const uint uint_qh = data_a_packed32[ib].qh;
113
+ const uvec2 qh0 = uvec2(((uint_qh >> 4*iqs) << 4) & 0x10, (uint_qh >> (4*iqs + 12)) & 0x10);
114
+ const uvec2 qh1 = uvec2(((uint_qh >> (4*iqs + 1)) << 4) & 0x10, (uint_qh >> (4*iqs + 13)) & 0x10);
115
+ const uvec2 qh2 = uvec2(((uint_qh >> (4*iqs + 2)) << 4) & 0x10, (uint_qh >> (4*iqs + 14)) & 0x10);
116
+ const uvec2 qh3 = uvec2(((uint_qh >> (4*iqs + 3)) << 4) & 0x10, (uint_qh >> (4*iqs + 15)) & 0x10);
117
+
118
+ const uint vui = data_a_packed32[ib].qs[iqs];
119
+ const vec4 v0 = vec4((vui & 0xF) | qh0.x, ((vui >> 4) & 0xF) | qh0.y, ((vui >> 8) & 0xF) | qh1.x, ((vui >> 12) & 0xF) | qh1.y) * dm.x + dm.y;
120
+ const vec4 v1 = vec4(((vui >> 16) & 0xF) | qh2.x, ((vui >> 20) & 0xF) | qh2.y, ((vui >> 24) & 0xF) | qh3.x, ((vui >> 28) & 0xF) | qh3.y) * dm.x + dm.y;
121
+
122
+ buf_a[buf_idx ] = FLOAT_TYPEV2(v0.xz);
123
+ buf_a[buf_idx + 1] = FLOAT_TYPEV2(v1.xz);
124
+ buf_a[buf_idx + 8] = FLOAT_TYPEV2(v0.yw);
125
+ buf_a[buf_idx + 9] = FLOAT_TYPEV2(v1.yw);
126
+ #elif defined(DATA_A_Q8_0)
127
+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
128
+ const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
129
+
130
+ const uint ib = idx / 8;
131
+ const uint iqs = idx & 0x07;
132
+
133
+ const float d = float(data_a_packed16[ib].d);
134
+ const i8vec2 v0 = unpack8(int32_t(data_a_packed16[ib].qs[2*iqs])).xy; // vec4 used due to #12147
135
+ const i8vec2 v1 = unpack8(int32_t(data_a_packed16[ib].qs[2*iqs + 1])).xy;
136
+ const vec4 v = vec4(v0.x, v0.y, v1.x, v1.y) * d;
137
+
138
+ buf_a[buf_idx ] = FLOAT_TYPEV2(v.xy);
139
+ buf_a[buf_idx + 1] = FLOAT_TYPEV2(v.zw);
140
+ #elif defined(DATA_A_Q1_0)
141
+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
142
+ const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
143
+
144
+ const uint ib = idx / 16;
145
+ const uint iqs = idx & 0xfu;
146
+
147
+ const float d = float(data_a[ib].d);
148
+ const uint bits = uint(data_a[ib].qs[iqs]);
149
+
150
+ buf_a[buf_idx ] = FLOAT_TYPEV2((bits & 0x01u) != 0u ? d : -d, (bits & 0x02u) != 0u ? d : -d);
151
+ buf_a[buf_idx + 1] = FLOAT_TYPEV2((bits & 0x04u) != 0u ? d : -d, (bits & 0x08u) != 0u ? d : -d);
152
+ buf_a[buf_idx + 2] = FLOAT_TYPEV2((bits & 0x10u) != 0u ? d : -d, (bits & 0x20u) != 0u ? d : -d);
153
+ buf_a[buf_idx + 3] = FLOAT_TYPEV2((bits & 0x40u) != 0u ? d : -d, (bits & 0x80u) != 0u ? d : -d);
154
+ #elif defined(DATA_A_Q2_K)
155
+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
156
+ const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
157
+
158
+ const uint ib = idx / 64; // 4 values per idx
159
+ const uint iqs = (idx % 64) * 2; // 0,2,4..126
160
+
161
+ const uint qsi = (iqs / 64) * 16 + (iqs % 16); // 0..15
162
+ const uint scalesi = iqs / 8; // 0..15
163
+ const uint qsshift = ((iqs % 64) / 16) * 2; // 0,2,4,6
164
+
165
+ const vec4 qs = vec4(unpack8((data_a_packed32[ib].qs[qsi / 2] >> qsshift) & 0x03030303));
166
+ const uint scales = data_a[ib].scales[scalesi];
167
+ const vec2 dm = vec2(data_a[ib].dm);
168
+
169
+ const vec4 v = dm.x * float(scales & 0xF) * qs - dm.y * float(scales >> 4);
170
+
171
+ buf_a[buf_idx ] = FLOAT_TYPEV2(v.xy);
172
+ buf_a[buf_idx + 1] = FLOAT_TYPEV2(v.zw);
173
+ #elif defined(DATA_A_Q3_K)
174
+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
175
+ const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
176
+
177
+ const uint ib = idx / 128; // 2 values per idx
178
+ const uint iqs = idx % 128; // 0..127
179
+
180
+ const uint n = iqs / 64; // 0,1
181
+ const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..62
182
+ const uint hmi = (iqs % 16) * 2; // 0,2,4..30
183
+ const uint j = (iqs % 64) / 4; // 0..3
184
+ const uint is = iqs / 8; // 0..15
185
+ const uint halfsplit = ((iqs % 64) / 16); // 0,1,2,3
186
+ const uint qsshift = halfsplit * 2; // 0,2,4,6
187
+
188
+ const int8_t us = int8_t(((data_a[ib].scales[is % 8] >> (4 * int(is / 8))) & 0xF)
189
+ | (((data_a[ib].scales[8 + (is % 4)] >> (2 * int(is / 4))) & 3) << 4));
190
+ const float dl = float(data_a[ib].d) * float(us - 32);
191
+
192
+ const vec2 qs = vec2(unpack8((uint(data_a_packed16[ib].qs[qsi / 2]) >> qsshift) & 0x0303).xy);
193
+ const vec2 hm = vec2(unpack8(((uint(data_a_packed16[ib].hmask[hmi / 2]) >> (4 * n + halfsplit)) & 0x0101 ^ 0x0101) << 2).xy);
194
+
195
+ buf_a[buf_idx] = FLOAT_TYPEV2(dl * (qs.x - hm.x),
196
+ dl * (qs.y - hm.y));
197
+ #elif defined(DATA_A_Q4_K)
198
+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
199
+ const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
200
+
201
+ const uint ib = idx / 64; // 4 values per idx
202
+ const uint iqs = (idx % 64) * 2; // 0,2,4..126
203
+
204
+ const uint n = iqs / 32; // 0,1,2,3
205
+ const uint b = (iqs % 32) / 16; // 0,1
206
+ const uint is = 2 * n + b; // 0..7
207
+ const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..126
208
+
209
+ const vec2 loadd = vec2(data_a[ib].dm);
210
+
211
+ const uvec3 scales = uvec3(data_a_packed32[ib].scales[0],
212
+ data_a_packed32[ib].scales[1],
213
+ data_a_packed32[ib].scales[2]);
214
+ const uint scalesoffs = (is & 3) * 8;
215
+
216
+ const uint scidx0 = (is < 4) ? 0 : 2;
217
+ const uint scidxshift0 = scalesoffs;
218
+ const uint scidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2;
219
+ const uint mbidx0 = (is < 4) ? 1 : 2;
220
+ const uint mbidxshift0 = (is < 4) ? scalesoffs : scalesoffs + 4;
221
+ const uint mbidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2;
222
+
223
+ const uint8_t sc = uint8_t(((scales[scidx0] >> scidxshift0) & 0xF) | ((scales[0] >> scidxshift1) & 0x30));
224
+ const uint8_t mbyte = uint8_t(((scales[mbidx0] >> mbidxshift0) & 0xF) | ((scales[1] >> mbidxshift1) & 0x30));
225
+
226
+ const float d = loadd.x * sc;
227
+ const float m = -loadd.y * mbyte;
228
+
229
+ const vec4 q = vec4(unpack8((data_a_packed32[ib].qs[qsi / 4] >> (b * 4)) & 0x0F0F0F0F));
230
+
231
+ buf_a[buf_idx ] = FLOAT_TYPEV2(fma(d, q.x, m), fma(d, q.y, m));
232
+ buf_a[buf_idx + 1] = FLOAT_TYPEV2(fma(d, q.z, m), fma(d, q.w, m));
233
+ #elif defined(DATA_A_Q5_K)
234
+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
235
+ const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
236
+
237
+ const uint ib = idx / 64; // 4 values per idx
238
+ const uint iqs = (idx % 64) * 2; // 0,2,4..126
239
+
240
+ const uint n = iqs / 32; // 0,1,2,3
241
+ const uint b = (iqs % 32) / 16; // 0,1
242
+ const uint is = 2 * n + b; // 0..7
243
+ const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..126
244
+ const uint qhi = (iqs % 16) * 2; // 0,2,4..30
245
+
246
+ const vec2 loadd = vec2(data_a[ib].dm);
247
+
248
+ const uvec3 scales = uvec3(data_a_packed32[ib].scales[0],
249
+ data_a_packed32[ib].scales[1],
250
+ data_a_packed32[ib].scales[2]);
251
+ const uint scalesoffs = (is & 3) * 8;
252
+
253
+ const uint scidx0 = (is < 4) ? 0 : 2;
254
+ const uint scidxshift0 = scalesoffs;
255
+ const uint scidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2;
256
+ const uint mbidx0 = (is < 4) ? 1 : 2;
257
+ const uint mbidxshift0 = (is < 4) ? scalesoffs : scalesoffs + 4;
258
+ const uint mbidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2;
259
+
260
+ const uint8_t sc = uint8_t(((scales[scidx0] >> scidxshift0) & 0xF) | ((scales[0] >> scidxshift1) & 0x30));
261
+ const uint8_t mbyte = uint8_t(((scales[mbidx0] >> mbidxshift0) & 0xF) | ((scales[1] >> mbidxshift1) & 0x30));
262
+
263
+ const float d = loadd.x * sc;
264
+ const float m = -loadd.y * mbyte;
265
+
266
+ const uint qs = (data_a_packed32[ib].qs[qsi / 4] >> (b * 4)) & 0x0F0F0F0F;
267
+ const uint qh = ((data_a_packed32[ib].qh[qhi / 4] >> (iqs / 16)) & 0x01010101) << 4;
268
+ const vec4 q = vec4(unpack8(qs | qh));
269
+
270
+ buf_a[buf_idx ] = FLOAT_TYPEV2(fma(d, q.x, m), fma(d, q.y, m));
271
+ buf_a[buf_idx + 1] = FLOAT_TYPEV2(fma(d, q.z, m), fma(d, q.w, m));
272
+ #elif defined(DATA_A_Q6_K)
273
+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
274
+ const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
275
+
276
+ const uint ib = idx / 128; // 2 values per idx
277
+ const uint iqs = idx % 128; // 0..127
278
+
279
+ const uint n = iqs / 64; // 0,1
280
+ const uint b = ((iqs % 64) / 32) * 4; // 0,4
281
+ const uint is_b = (iqs % 16) / 8; // 0,1
282
+ const uint qhshift = ((iqs % 64) / 16) * 2; // 0,2,4,6
283
+ const uint is = 8 * n + qhshift + is_b; // 0..15
284
+ const uint qsi = n * 32 + (iqs % 32); // 0..63
285
+ const uint qhi = n * 16 + (iqs % 16); // 0..31
286
+
287
+ const float dscale = float(data_a[ib].d) * float(data_a[ib].scales[is]);
288
+
289
+ const uint ql = (uint(data_a_packed16[ib].ql[qsi]) >> b) & 0x0F0F;
290
+ const uint qh = (uint(data_a_packed16[ib].qh[qhi]) >> qhshift) & 0x0303;
291
+ const vec2 q = (vec2(unpack8(ql | (qh << 4)).xy) - 32) * dscale;
292
+
293
+ buf_a[buf_idx] = FLOAT_TYPEV2(q.x, q.y);
294
+ #elif defined(DATA_A_IQ1_S)
295
+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
296
+ const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
297
+
298
+ const uint ib = idx / 32; // 8 values per idx
299
+ const uint ib32 = (idx % 32) / 4; // 0..7
300
+ const uint ib8 = idx % 32;
301
+
302
+ const float d = float(data_a[ib].d);
303
+ const uint qh = data_a[ib].qh[ib32];
304
+ const uint qs = data_a[ib].qs[ib8];
305
+ const float dl = d * (2 * bitfieldExtract(qh, 12, 3) + 1);
306
+ const float delta = ((qh & 0x8000) != 0) ? -IQ1S_DELTA : IQ1S_DELTA;
307
+ const int16_t grid = int16_t(iq1s_grid[qs | (bitfieldExtract(qh, 3 * int(ib8 & 3), 3) << 8)]);
308
+
309
+ [[unroll]] for (int k = 0; k < 4; ++k) {
310
+ buf_a[buf_idx + k] = FLOAT_TYPEV2(dl * (bitfieldExtract(grid, 4 * k , 2) + delta),
311
+ dl * (bitfieldExtract(grid, 4 * k + 2, 2) + delta));
312
+ }
313
+ #elif defined(DATA_A_IQ1_M)
314
+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
315
+ const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
316
+
317
+ const uint ib = idx / 32; // 8 values per idx
318
+ const uint ib8 = idx % 32;
319
+ const uint ib16 = ib8 / 2;
320
+
321
+ const uint16_t[4] scales = data_a[ib].scales;
322
+ const u16vec4 s = u16vec4(scales[0], scales[1], scales[2], scales[3]) >> 12;
323
+ const float d = float(unpackHalf2x16(s.x | (s.y << 4) | (s.z << 8) | (s.w << 12)).x);
324
+ const uint sc = scales[ib8 / 8];
325
+ const uint qs = data_a[ib].qs[ib8];
326
+ const uint qh = data_a[ib].qh[ib16] >> (4 * (ib8 & 1));
327
+ const float dl = d * (2 * bitfieldExtract(sc, 3 * int(ib16 & 3), 3) + 1);
328
+ const float delta = ((qh & 8) != 0) ? -IQ1M_DELTA : IQ1M_DELTA;
329
+ const int16_t grid = int16_t(iq1s_grid[qs | ((qh & 7) << 8)]);
330
+
331
+ [[unroll]] for (int k = 0; k < 4; ++k) {
332
+ buf_a[buf_idx + k] = FLOAT_TYPEV2(dl * (bitfieldExtract(grid, 4 * k , 2) + delta),
333
+ dl * (bitfieldExtract(grid, 4 * k + 2, 2) + delta));
334
+ }
335
+ #elif defined(DATA_A_IQ2_XXS)
336
+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
337
+ const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
338
+
339
+ const uint ib = idx / 32; // 8 values per idx
340
+ const uint ib32 = (idx % 32) / 4; // 0..7
341
+ const uint ib8 = idx % 4;
342
+
343
+ const float d = float(data_a[ib].d);
344
+ const uint qs = data_a[ib].qs[8 * ib32 + ib8];
345
+ const uint signs = pack32(u8vec4(
346
+ data_a[ib].qs[8*ib32 + 4],
347
+ data_a[ib].qs[8*ib32 + 5],
348
+ data_a[ib].qs[8*ib32 + 6],
349
+ data_a[ib].qs[8*ib32 + 7]
350
+ ));
351
+ const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + (signs >> 28)));
352
+ const uint32_t sign7 = bitfieldExtract(signs, 7 * int(ib8), 7);
353
+ const uint sign = sign7 | (bitCount(sign7) << 7);
354
+ const uvec2 grid = iq2xxs_grid[qs];
355
+ const vec4 grid0 = vec4(unpack8(grid.x));
356
+ const vec4 grid1 = vec4(unpack8(grid.y));
357
+
358
+ buf_a[buf_idx ] = db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x,
359
+ (sign & 2) != 0 ? -grid0.y : grid0.y);
360
+ buf_a[buf_idx + 1] = db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z,
361
+ (sign & 8) != 0 ? -grid0.w : grid0.w);
362
+ buf_a[buf_idx + 2] = db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x,
363
+ (sign & 32) != 0 ? -grid1.y : grid1.y);
364
+ buf_a[buf_idx + 3] = db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z,
365
+ (sign & 128) != 0 ? -grid1.w : grid1.w);
366
+ #elif defined(DATA_A_IQ2_XS)
367
+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
368
+ const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
369
+
370
+ const uint ib = idx / 32; // 8 values per idx
371
+ const uint ib32 = (idx % 32) / 4; // 0..7
372
+ const uint ib8 = idx % 4; // 0..3
373
+
374
+ const float d = float(data_a[ib].d);
375
+ const uint scale = (data_a[ib].scales[ib32] >> (2 * (ib8 & 2))) & 0xf;
376
+ const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + scale));
377
+ const uint qs = data_a[ib].qs[4 * ib32 + ib8];
378
+ const uint sign7 = qs >> 9;
379
+ const uint sign = sign7 | (bitCount(sign7) << 7);
380
+ const uvec2 grid = iq2xs_grid[qs & 511];
381
+ const vec4 grid0 = vec4(unpack8(grid.x));
382
+ const vec4 grid1 = vec4(unpack8(grid.y));
383
+
384
+ buf_a[buf_idx ] = db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x,
385
+ (sign & 2) != 0 ? -grid0.y : grid0.y);
386
+ buf_a[buf_idx + 1] = db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z,
387
+ (sign & 8) != 0 ? -grid0.w : grid0.w);
388
+ buf_a[buf_idx + 2] = db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x,
389
+ (sign & 32) != 0 ? -grid1.y : grid1.y);
390
+ buf_a[buf_idx + 3] = db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z,
391
+ (sign & 128) != 0 ? -grid1.w : grid1.w);
392
+ #elif defined(DATA_A_IQ2_S)
393
+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
394
+ const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
395
+
396
+ const uint ib = idx / 32; // 8 values per idx
397
+ const uint ib8 = idx % 32; // 0..31
398
+ const uint ib32 = ib8 / 4; // 0..7
399
+
400
+ const uint scale = (data_a[ib].scales[ib32] >> (2 * (ib8 & 2))) & 0xf;
401
+ const uint qs = data_a[ib].qs[ib8];
402
+ const uint qh = data_a[ib].qh[ib32];
403
+ const uint qhshift = 2 * (ib8 % 4);
404
+ const uint sign = data_a[ib].qs[QUANT_K / 8 + ib8];
405
+
406
+ const float d = float(data_a[ib].d);
407
+ const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + scale));
408
+ const uvec2 grid = iq2s_grid[qs | ((qh << (8 - qhshift)) & 0x300)];
409
+ const vec4 grid0 = vec4(unpack8(grid.x));
410
+ const vec4 grid1 = vec4(unpack8(grid.y));
411
+
412
+ buf_a[buf_idx ] = db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x,
413
+ (sign & 2) != 0 ? -grid0.y : grid0.y);
414
+ buf_a[buf_idx + 1] = db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z,
415
+ (sign & 8) != 0 ? -grid0.w : grid0.w);
416
+ buf_a[buf_idx + 2] = db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x,
417
+ (sign & 32) != 0 ? -grid1.y : grid1.y);
418
+ buf_a[buf_idx + 3] = db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z,
419
+ (sign & 128) != 0 ? -grid1.w : grid1.w);
420
+ #elif defined(DATA_A_IQ3_XXS)
421
+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
422
+ const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
423
+
424
+ const uint ib = idx / 64; // 4 values per idx
425
+ const uint iqs = idx % 64; // 0..63
426
+ const uint is = QUANT_K / 4 + 4 * (iqs / 8); // 8 values
427
+
428
+ const float d = float(data_a[ib].d);
429
+ const uint qs = data_a[ib].qs[iqs];
430
+ const uint signs = pack32(u16vec2(
431
+ data_a_packed16[ib].qs[is/2],
432
+ data_a_packed16[ib].qs[is/2+1]
433
+ ));
434
+ const float db = d * 0.5 * (0.5 + (signs >> 28));
435
+ const uint32_t sign7 = bitfieldExtract(signs, 7 * (int(iqs / 2) % 4), 7);
436
+ const uint sign = (sign7 | (bitCount(sign7) << 7)) >> (4 * (idx % 2));
437
+ const uint grid = iq3xxs_grid[qs];
438
+ const vec4 v = db * vec4(unpack8(grid));
439
+
440
+ buf_a[buf_idx ] = FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x,
441
+ (sign & 2) != 0 ? -v.y : v.y);
442
+ buf_a[buf_idx + 1] = FLOAT_TYPEV2((sign & 4) != 0 ? -v.z : v.z,
443
+ (sign & 8) != 0 ? -v.w : v.w);
444
+ #elif defined(DATA_A_IQ3_S)
445
+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
446
+ const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
447
+
448
+ const uint ib = idx / 64; // 4 values per idx
449
+ const uint iqs = idx % 64; // 0..63
450
+ const uint iqh = iqs / 8;
451
+
452
+ const float d = float(data_a[ib].d);
453
+ const uint qs = data_a[ib].qs[iqs];
454
+ const uint qh = data_a[ib].qh[iqh];
455
+ const int8_t sign = int8_t(data_a[ib].signs[iqs / 2] >> (4 * (idx % 2)));
456
+ const uint scale = data_a[ib].scales[iqs / 16];
457
+ const i8vec2 sign01 = i8vec2(1 - (2 & i8vec2(sign << 1, sign)));
458
+ const float db = d * (1 + 2 * ((scale >> (4 * (iqh & 1))) & 0xf));
459
+ const uint32_t grid = iq3s_grid[qs | ((qh << (8 - (iqs % 8))) & 256)];
460
+ const vec4 v = db * vec4(unpack8(grid));
461
+
462
+ buf_a[buf_idx ] = FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x,
463
+ (sign & 2) != 0 ? -v.y : v.y);
464
+ buf_a[buf_idx + 1] = FLOAT_TYPEV2((sign & 4) != 0 ? -v.z : v.z,
465
+ (sign & 8) != 0 ? -v.w : v.w);
466
+ #elif defined(DATA_A_IQ4_XS)
467
+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
468
+ const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
469
+
470
+ const uint ib = idx / 64; // 4 values per idx
471
+ const uint ib32 = (idx % 64) / 8; // 0..7
472
+ const uint iq = 4 * ib32 + (idx % 4);
473
+
474
+ const uint sl = (data_a[ib].scales_l[ib32/2] >> (4 * (ib32 & 1))) & 0xF;
475
+ const uint sh = ((data_a[ib].scales_h) >> (2 * ib32)) & 3;
476
+ const uint qshift = idx & 4;
477
+ u8vec4 qs = unpack8((uint(data_a_packed32[ib].qs[iq]) >> qshift) & 0x0F0F0F0F);
478
+
479
+ const float d = float(data_a[ib].d);
480
+ const vec4 v = d * float(int(sl | (sh << 4)) - 32) * vec4(kvalues_iq4nl[qs.x], kvalues_iq4nl[qs.y], kvalues_iq4nl[qs.z], kvalues_iq4nl[qs.w]);
481
+
482
+ buf_a[buf_idx ] = FLOAT_TYPEV2(v.xy);
483
+ buf_a[buf_idx + 1] = FLOAT_TYPEV2(v.zw);
484
+ #elif defined(DATA_A_IQ4_NL)
485
+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
486
+ const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 4;
487
+
488
+ const uint ib = idx / 8;
489
+ const uint iqs = idx & 0x07;
490
+
491
+ const FLOAT_TYPE d = FLOAT_TYPE(data_a_packed16[ib].d);
492
+ const uint vui = uint(data_a_packed16[ib].qs[iqs]);
493
+
494
+ buf_a[buf_idx ] = d * FLOAT_TYPEV2(kvalues_iq4nl[vui & 0xF],
495
+ kvalues_iq4nl[bitfieldExtract(vui, 8, 4)]);
496
+ buf_a[buf_idx + 8] = d * FLOAT_TYPEV2(kvalues_iq4nl[bitfieldExtract(vui, 4, 4)],
497
+ kvalues_iq4nl[vui >> 12]);
498
+ #elif defined(DATA_A_MXFP4)
499
+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
500
+ const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 4;
501
+
502
+ const uint ib = idx / 8;
503
+ const uint iqs = (idx & 0x07) * 2;
504
+
505
+ const float d = e8m0_to_fp32(data_a[ib].e) * 0.5;
506
+ const uint vui = uint(data_a[ib].qs[iqs]);
507
+ const uint vui2 = uint(data_a[ib].qs[iqs+1]);
508
+
509
+ buf_a[buf_idx ] = FLOAT_TYPEV2(kvalues_mxfp4[vui & 0xF] * d,
510
+ kvalues_mxfp4[vui2 & 0xF] * d);
511
+ buf_a[buf_idx + 8] = FLOAT_TYPEV2(kvalues_mxfp4[vui >> 4] * d,
512
+ kvalues_mxfp4[vui2 >> 4] * d);
513
+ #elif defined(DATA_A_NVFP4)
514
+ const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
515
+ // lo and hi nibbles are 8 elements apart, which doesn't quite line up with
516
+ // how the thread mapping and buf_idx calculation works for other types.
517
+ const uint buf_idx = col * SHMEM_STRIDE + (row & 3) + (row & ~3) * 2;
518
+
519
+ const uint ib = idx / 16u;
520
+ const uint sub = (idx & 0xC) >> 2;
521
+ const uint iqs = (idx & 0xF) * 2;
522
+ const float d = ue4m3_to_fp32(data_a[ib].d[sub]) * 0.5;
523
+ const uint vui = uint(data_a[ib].qs[iqs]);
524
+ const uint vui2 = uint(data_a[ib].qs[iqs+1]);
525
+
526
+ buf_a[buf_idx ] = FLOAT_TYPEV2(kvalues_mxfp4[vui & 0xF] * d,
527
+ kvalues_mxfp4[vui2 & 0xF] * d);
528
+ buf_a[buf_idx + 4] = FLOAT_TYPEV2(kvalues_mxfp4[vui >> 4] * d,
529
+ kvalues_mxfp4[vui2 >> 4] * d);
530
+ #endif
531
+ }
532
+
533
+ #if !defined(MUL_MAT_ID)
534
+ void load_b_to_shmem(const uint pos_b, const uint row, const uint col, const uint idx_n, const uint block, const uint end_k) {
535
+ #if LOAD_VEC_B == 8
536
+ if (ALIGNED != 0) {
537
+ // Not supported for b_type bf16 because bf16mat2x4 does not exist
538
+ const uint idx = pos_b + col * p.stride_b / LOAD_VEC_B + row;
539
+ const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2;
540
+ FLOAT_TYPEV8 bb = FLOAT_TYPEV8(data_b[idx]);
541
+ buf_b[buf_idx + 0] = bb[0].xy;
542
+ buf_b[buf_idx + 1] = bb[0].zw;
543
+ buf_b[buf_idx + 2] = bb[1].xy;
544
+ buf_b[buf_idx + 3] = bb[1].zw;
545
+ return;
546
+ }
547
+ #elif LOAD_VEC_B == 4
548
+ if (ALIGNED != 0) {
549
+ const uint idx = pos_b + col * p.stride_b / LOAD_VEC_B + row;
550
+ const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2;
551
+ #if defined(DATA_B_BF16)
552
+ FLOAT_TYPEV4 bb = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_b[idx]));
553
+ #else
554
+ FLOAT_TYPEV4 bb = FLOAT_TYPEV4(data_b[idx]);
555
+ #endif
556
+ buf_b[buf_idx + 0] = bb.xy;
557
+ buf_b[buf_idx + 1] = bb.zw;
558
+ return;
559
+ }
560
+ #endif
561
+ const uint idx = pos_b + col * p.stride_b + row * 2;
562
+ const uint buf_idx = col * SHMEM_STRIDE + row;
563
+ if (idx_n < p.N && block + row * 2 + 1 < end_k) {
564
+ buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]),
565
+ TO_FLOAT_TYPE(data_b_scalar[idx + 1]));
566
+ } else if (idx_n < p.N && block + row * 2 < end_k) {
567
+ buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), 0.0f);
568
+ } else {
569
+ buf_b[buf_idx] = FLOAT_TYPEV2(0.0f);
570
+ }
571
+ }
572
+ #else
573
+ void load_b_to_shmem(const uint pos_b, const uint row, const uint col, const uint ic, const uint _ne1, const uint block, const uint end_k) {
574
+ #if LOAD_VEC_B == 8
575
+ if (ALIGNED != 0) {
576
+ // Not supported for b_type bf16 because bf16mat2x4 does not exist
577
+ const u16vec2 row_idx = row_ids[col];
578
+ const uint idx = pos_b + row_idx.y * p.batch_stride_b / LOAD_VEC_B + (row_idx.x % p.ne11) * p.stride_b / LOAD_VEC_B + row;
579
+ const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2;
580
+ FLOAT_TYPEV8 bb = FLOAT_TYPEV8(data_b[idx]);
581
+ buf_b[buf_idx + 0] = bb[0].xy;
582
+ buf_b[buf_idx + 1] = bb[0].zw;
583
+ buf_b[buf_idx + 2] = bb[1].xy;
584
+ buf_b[buf_idx + 3] = bb[1].zw;
585
+ return;
586
+ }
587
+ #elif LOAD_VEC_B == 4
588
+ if (ALIGNED != 0) {
589
+ const u16vec2 row_idx = row_ids[col];
590
+ const uint idx = pos_b + row_idx.y * p.batch_stride_b / LOAD_VEC_B + (row_idx.x % p.ne11) * p.stride_b / LOAD_VEC_B + row;
591
+ const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2;
592
+ #if defined(DATA_B_BF16)
593
+ FLOAT_TYPEV4 bb = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_b[idx]));
594
+ #else
595
+ FLOAT_TYPEV4 bb = FLOAT_TYPEV4(data_b[idx]);
596
+ #endif
597
+ buf_b[buf_idx + 0] = bb.xy;
598
+ buf_b[buf_idx + 1] = bb.zw;
599
+ return;
600
+ }
601
+ #endif
602
+ const uint row_i = ic * BN + col;
603
+ const uint buf_idx = col * SHMEM_STRIDE + row;
604
+ if (row_i < _ne1 && block + row * 2 + 1 < end_k) {
605
+ const u16vec2 row_idx = row_ids[col];
606
+ const uint idx = pos_b + row_idx.y * p.batch_stride_b + (row_idx.x % p.ne11) * p.stride_b + row * 2;
607
+ buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]),
608
+ TO_FLOAT_TYPE(data_b_scalar[idx + 1]));
609
+ } else if (row_i < _ne1 && block + row * 2 < end_k) {
610
+ const u16vec2 row_idx = row_ids[col];
611
+ const uint idx = pos_b + row_idx.y * p.batch_stride_b + (row_idx.x % p.ne11) * p.stride_b + row * 2;
612
+ buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), 0.0f);
613
+ } else {
614
+ buf_b[buf_idx] = FLOAT_TYPEV2(0.0f);
615
+ }
616
+ }
617
+ #endif
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #ifdef MUL_MAT_ID
2
+ shared u16vec2 row_ids[BN];
3
+ uint _ne1;
4
+
5
+ #ifdef MUL_MAT_ID_USE_SUBGROUPS
6
+ shared uvec4 ballots_sh[NUM_WARPS];
7
+
8
+ void load_row_ids(uint expert_idx, bool nei0_is_pow2, uint ic) {
9
+ _ne1 = 0;
10
+ uint num_elements = p.nei1 * p.nei0;
11
+ uint nei0shift = findLSB(p.nei0);
12
+
13
+ uint ids[16];
14
+ uint iter = 0;
15
+
16
+ uint expert_count = data_expert_count[expert_idx];
17
+
18
+ for (uint j = 0; j < num_elements; j += BLOCK_SIZE) {
19
+ // prefetch up to 16 elements
20
+ if (iter == 0) {
21
+ [[unroll]] for (uint k = 0; k < 16; ++k) {
22
+ uint i = j + gl_LocalInvocationIndex + k*BLOCK_SIZE;
23
+ bool in_range = i < num_elements;
24
+ uint ii1;
25
+ if (nei0_is_pow2) {
26
+ ii1 = i >> nei0shift;
27
+ } else {
28
+ ii1 = i / p.nei0;
29
+ }
30
+ uint ii0 = i - ii1 * p.nei0;
31
+ ids[k] = in_range ? data_ids[ii1*p.nbi1 + ii0] : 0;
32
+ }
33
+ }
34
+ uint i = j + gl_LocalInvocationIndex;
35
+ bool in_range = i < num_elements;
36
+ uint ii1;
37
+ if (nei0_is_pow2) {
38
+ ii1 = i >> nei0shift;
39
+ } else {
40
+ ii1 = i / p.nei0;
41
+ }
42
+ uint ii0 = i - ii1 * p.nei0;
43
+ uint id = ids[iter++];
44
+ uvec4 ballot = subgroupBallot(in_range && id == expert_idx);
45
+
46
+ if (gl_SubgroupInvocationID == 0) {
47
+ ballots_sh[gl_SubgroupID] = ballot;
48
+ }
49
+ barrier();
50
+
51
+ uint subgroup_base = 0;
52
+ uint total = 0;
53
+ for (uint k = 0; k < gl_NumSubgroups; ++k) {
54
+ if (k == gl_SubgroupID) {
55
+ subgroup_base = total;
56
+ }
57
+ total += subgroupBallotBitCount(ballots_sh[k]);
58
+ }
59
+ barrier();
60
+
61
+ uint idx = subgroup_base + subgroupBallotExclusiveBitCount(ballot);
62
+ if (in_range && id == expert_idx && _ne1 + idx >= ic * BN && _ne1 + idx < (ic + 1) * BN) {
63
+ row_ids[_ne1 + idx - ic * BN] = u16vec2(ii0, ii1);
64
+ }
65
+ _ne1 += total;
66
+ iter &= 15;
67
+ if (_ne1 >= (ic + 1) * BN || _ne1 == expert_count) {
68
+ break;
69
+ }
70
+ }
71
+ barrier();
72
+ }
73
+ #endif // MUL_MAT_ID_USE_SUBGROUPS
74
+ #endif // MUL_MAT_ID
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp ADDED
@@ -0,0 +1,311 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+
3
+ #extension GL_EXT_control_flow_attributes : enable
4
+ #extension GL_EXT_shader_16bit_storage : require
5
+ #extension GL_EXT_shader_explicit_arithmetic_types_int8 : require
6
+
7
+ #extension GL_EXT_integer_dot_product : require
8
+
9
+ #ifdef FLOAT16
10
+ #extension GL_EXT_shader_explicit_arithmetic_types_float16 : require
11
+ #endif
12
+
13
+ #if defined(MUL_MAT_ID_USE_SUBGROUPS)
14
+ #extension GL_KHR_shader_subgroup_basic : enable
15
+ #extension GL_KHR_shader_subgroup_ballot : enable
16
+ #endif
17
+
18
+ #ifdef MUL_MAT_ID
19
+ #extension GL_EXT_shader_explicit_arithmetic_types_int16 : require
20
+ #endif
21
+
22
+ #include "types.glsl"
23
+
24
+ layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
25
+
26
+ layout (binding = 0) readonly buffer A {A_TYPE data_a[];};
27
+ #if defined(A_TYPE_PACKED16)
28
+ layout (binding = 0) readonly buffer A_PACKED16 {A_TYPE_PACKED16 data_a_packed16[];};
29
+ #endif
30
+ #if defined(A_TYPE_PACKED32)
31
+ layout (binding = 0) readonly buffer A_PACKED32 {A_TYPE_PACKED32 data_a_packed32[];};
32
+ #endif
33
+ layout (binding = 1) readonly buffer B {block_q8_1_x4_packed128 data_b[];};
34
+ layout (binding = 2) writeonly buffer D {D_TYPE data_d[];};
35
+
36
+ #ifdef MUL_MAT_ID
37
+ layout (binding = 3) readonly buffer IDS {int data_ids[];};
38
+ layout (binding = 4) readonly buffer Counts {int data_expert_count[];};
39
+ #endif
40
+
41
+ layout (push_constant) uniform parameter
42
+ {
43
+ uint M;
44
+ uint N;
45
+ uint K;
46
+ uint stride_a;
47
+ uint stride_b;
48
+ uint stride_d;
49
+
50
+ uint batch_stride_a;
51
+ uint batch_stride_b;
52
+ uint batch_stride_d;
53
+
54
+ #ifdef MUL_MAT_ID
55
+ uint nei0;
56
+ uint nei1;
57
+ uint nbi1;
58
+ uint ne11;
59
+ #else
60
+ uint base_work_group_z;
61
+ uint num_batches;
62
+ uint k_split;
63
+ uint ne02;
64
+ uint ne12;
65
+ uint broadcast2;
66
+ uint broadcast3;
67
+ #endif
68
+ } p;
69
+
70
+ layout (constant_id = 0) const uint BLOCK_SIZE = 64;
71
+ layout (constant_id = 1) const uint BM = 64;
72
+ layout (constant_id = 2) const uint BN = 64;
73
+ // layout (constant_id = 3) const uint BK = 32;
74
+ layout (constant_id = 4) const uint WM = 32;
75
+ layout (constant_id = 5) const uint WN = 32;
76
+ layout (constant_id = 6) const uint WMITER = 2;
77
+ layout (constant_id = 7) const uint TM = 4;
78
+ layout (constant_id = 8) const uint TN = 2;
79
+ layout (constant_id = 9) const uint TK = 1; // Only needed for coopmat
80
+ layout (constant_id = 10) const uint WARP = 32;
81
+
82
+ #define BK 32
83
+
84
+ #include "mul_mmq_shmem_types.glsl"
85
+
86
+ #ifdef MUL_MAT_ID
87
+ #define BK_STEP 1
88
+ #else
89
+ #ifndef BK_STEP
90
+ #define BK_STEP 4
91
+ #endif
92
+ #endif
93
+
94
+ // Shared memory cache
95
+ shared block_a_cache buf_a[BM * BK_STEP];
96
+ shared block_b_cache buf_b[BN * BK_STEP];
97
+ // Register cache
98
+ block_a_cache cache_a[WMITER * TM];
99
+ block_b_cache cache_b;
100
+
101
+ #define LOAD_VEC_A (4 * QUANT_R_MMQ)
102
+ #define LOAD_VEC_B 16
103
+
104
+ #define NUM_WARPS (BLOCK_SIZE / WARP)
105
+
106
+ #include "mul_mm_id_funcs.glsl"
107
+ #include "mul_mmq_funcs.glsl"
108
+
109
+ void main() {
110
+ const uint ic = gl_WorkGroupID.y;
111
+
112
+ #ifdef MUL_MAT_ID
113
+ const uint expert_idx = gl_WorkGroupID.z;
114
+ if (ic * BN >= data_expert_count[expert_idx]) {
115
+ return;
116
+ }
117
+ #endif
118
+ #ifdef NEEDS_INIT_IQ_SHMEM
119
+ init_iq_shmem(gl_WorkGroupSize);
120
+ #endif
121
+
122
+ #ifndef MUL_MAT_ID
123
+ const uint batch_idx = gl_WorkGroupID.z + p.base_work_group_z;
124
+
125
+ const uint i13 = batch_idx / p.ne12;
126
+ const uint i12 = batch_idx % p.ne12;
127
+
128
+ const uint i03 = i13 / p.broadcast3;
129
+ const uint i02 = i12 / p.broadcast2;
130
+
131
+ const uint batch_idx_a = i03 * p.ne02 + i02;
132
+ #endif
133
+
134
+ const uint blocks_m = (p.M + BM - 1) / BM;
135
+ const uint ir = gl_WorkGroupID.x % blocks_m;
136
+ const uint ik = gl_WorkGroupID.x / blocks_m;
137
+
138
+ const uint WNITER = (WM * WN) / (WARP * TM * TN * WMITER);
139
+ const uint WSUBM = WM / WMITER;
140
+ const uint WSUBN = WN / WNITER;
141
+ const uint warp_i = gl_LocalInvocationID.x / WARP;
142
+
143
+ const uint tiw = gl_LocalInvocationID.x % WARP;
144
+
145
+ const uint tiwr = tiw % (WSUBM / TM);
146
+ const uint tiwc = tiw / (WSUBM / TM);
147
+
148
+ const uint warp_r = warp_i % (BM / WM);
149
+ const uint warp_c = warp_i / (BM / WM);
150
+
151
+ const uint loadr_a = gl_LocalInvocationID.x % (BK / LOAD_VEC_A);
152
+ const uint loadc_a = gl_LocalInvocationID.x / (BK / LOAD_VEC_A);
153
+ const uint loadr_b = gl_LocalInvocationID.x % (BK / LOAD_VEC_B);
154
+ const uint loadc_b = gl_LocalInvocationID.x / (BK / LOAD_VEC_B);
155
+
156
+ const uint loadstride_a = BLOCK_SIZE * LOAD_VEC_A / BK;
157
+ const uint loadstride_b = BLOCK_SIZE * LOAD_VEC_B / BK;
158
+
159
+ #ifdef MUL_MAT_ID
160
+ #ifdef MUL_MAT_ID_USE_SUBGROUPS
161
+ if (bitCount(p.nei0) == 1) {
162
+ load_row_ids(expert_idx, true, ic);
163
+ } else {
164
+ load_row_ids(expert_idx, false, ic);
165
+ }
166
+ #else
167
+ _ne1 = 0;
168
+ for (uint ii1 = 0; ii1 < p.nei1 && _ne1 < (ic + 1) * BN; ii1++) {
169
+ for (uint ii0 = 0; ii0 < p.nei0 && _ne1 < (ic + 1) * BN; ii0++) {
170
+ if (data_ids[ii1*p.nbi1 + ii0] == expert_idx) {
171
+ if (_ne1 >= ic * BN) {
172
+ row_ids[_ne1 - ic * BN] = u16vec2(ii0, ii1);
173
+ }
174
+ _ne1++;
175
+ }
176
+ }
177
+ }
178
+
179
+ barrier();
180
+ #endif
181
+
182
+ // Workgroup has no work
183
+ if (ic * BN >= _ne1) return;
184
+ #endif
185
+
186
+ #ifdef MUL_MAT_ID
187
+ const uint start_k = 0;
188
+ const uint end_k = p.K;
189
+ #else
190
+ const uint start_k = ik * p.k_split;
191
+ const uint end_k = min(p.K, (ik + 1) * p.k_split);
192
+ #endif
193
+
194
+ uint pos_a_ib =
195
+ #ifdef MUL_MAT_ID
196
+ expert_idx * (p.batch_stride_a / BK) +
197
+ #else
198
+ batch_idx_a * (p.batch_stride_a / BK) +
199
+ #endif
200
+ (ir * BM * p.stride_a + start_k) / BK;
201
+ #ifdef MUL_MAT_ID
202
+ uint pos_b_ib = 0;
203
+ #else
204
+ uint pos_b_ib = (batch_idx * p.batch_stride_b + ic * BN * p.stride_b + start_k) / BK;
205
+ #endif
206
+
207
+ ACC_TYPE sums[WMITER * TM * WNITER * TN];
208
+
209
+ [[unroll]] for (uint i = 0; i < WMITER*TM*WNITER*TN; i++) {
210
+ sums[i] = ACC_TYPE(0.0f);
211
+ }
212
+
213
+ for (uint block = start_k; block < end_k; block += BK * BK_STEP) {
214
+ [[unroll]] for (uint l = 0; loadc_a + l < BM; l += loadstride_a) {
215
+ const uint buf_ib = loadc_a + l;
216
+ const uint ib = pos_a_ib + buf_ib * p.stride_a / BK;
217
+ const uint iqs = loadr_a;
218
+
219
+ [[unroll]] for (uint k_step = 0; k_step < BK_STEP; k_step++) {
220
+ if (block + k_step * BK < end_k) {
221
+ block_a_to_shmem(k_step * BM + buf_ib, ib + k_step, iqs);
222
+ }
223
+ }
224
+ }
225
+ [[unroll]] for (uint l = 0; loadc_b + l < BN; l += loadstride_b) {
226
+ const uint buf_ib = loadc_b + l;
227
+
228
+ #ifdef MUL_MAT_ID
229
+ const u16vec2 row_idx = row_ids[buf_ib];
230
+ const uint ib = pos_b_ib + row_idx.y * p.batch_stride_b / BK + (row_idx.x % p.ne11) * p.stride_b / BK;
231
+ #else
232
+ const uint ib = pos_b_ib + buf_ib * p.stride_b / BK;
233
+ #endif
234
+ const uint iqs = loadr_b;
235
+
236
+ [[unroll]] for (uint k_step = 0; k_step < BK_STEP; k_step++) {
237
+ block_b_to_shmem(k_step * BN + buf_ib, ib + k_step, iqs, block + k_step * BK < end_k);
238
+ }
239
+ }
240
+
241
+ barrier();
242
+
243
+ pos_a_ib += BK_STEP;
244
+ pos_b_ib += BK_STEP;
245
+
246
+ for (uint k_step = 0; k_step < BK_STEP; k_step++) {
247
+ // Load from shared into cache
248
+ [[unroll]] for (uint wsir = 0; wsir < WMITER; wsir++) {
249
+ [[unroll]] for (uint cr = 0; cr < TM; cr++) {
250
+ const uint reg_ib = wsir * TM + cr;
251
+ const uint buf_ib = warp_r * WM + wsir * WSUBM + tiwr * TM + cr;
252
+
253
+ block_a_to_registers(reg_ib, k_step * BM + buf_ib);
254
+ }
255
+ }
256
+
257
+ [[unroll]] for (uint wsic = 0; wsic < WNITER; wsic++) {
258
+ [[unroll]] for (uint cc = 0; cc < TN; cc++) {
259
+ const uint ib = k_step * BN + warp_c * WN + wsic * WSUBN + tiwc * TN + cc;
260
+ block_b_to_registers(ib);
261
+
262
+ [[unroll]] for (uint wsir = 0; wsir < WMITER; wsir++) {
263
+ [[unroll]] for (uint cr = 0; cr < TM; cr++) {
264
+ const uint cache_a_idx = wsir * TM + cr;
265
+ const uint sums_idx = (wsic * TN + cc) * (WMITER * TM) + wsir * TM + cr;
266
+
267
+ sums[sums_idx] += mmq_dot_product(cache_a_idx);
268
+ }
269
+ }
270
+ }
271
+ }
272
+ }
273
+
274
+ barrier();
275
+ }
276
+
277
+ const uint dr = ir * BM + warp_r * WM;
278
+ const uint dc = ic * BN + warp_c * WN;
279
+
280
+ #ifndef MUL_MAT_ID
281
+ const uint offsets = batch_idx * p.batch_stride_d + ik * p.batch_stride_d * p.num_batches;
282
+ #endif
283
+
284
+ [[unroll]] for (uint wsic = 0; wsic < WNITER; wsic++) {
285
+ [[unroll]] for (uint wsir = 0; wsir < WMITER; wsir++) {
286
+
287
+ const uint dr_warp = dr + wsir * WSUBM + tiwr * TM;
288
+ const uint dc_warp = dc + wsic * WSUBN + tiwc * TN;
289
+ [[unroll]] for (uint cc = 0; cc < TN; cc++) {
290
+ #ifdef MUL_MAT_ID
291
+ const uint row_i = dc_warp + cc;
292
+ if (row_i >= _ne1) break;
293
+
294
+ const u16vec2 row_idx = row_ids[row_i - ic * BN];
295
+ #endif // MUL_MAT_ID
296
+ [[unroll]] for (uint cr = 0; cr < TM; cr++) {
297
+ const uint sums_idx = (wsic * TN + cc) * WMITER * TM + wsir * TM + cr;
298
+ #ifdef MUL_MAT_ID
299
+ if (dr_warp + cr < p.M) {
300
+ data_d[row_idx.y * p.batch_stride_d + row_idx.x * p.stride_d + dr_warp + cr] = D_TYPE(sums[sums_idx].x);
301
+ }
302
+ #else
303
+ if (dr_warp + cr < p.M && dc_warp + cc < p.N) {
304
+ data_d[offsets + (dc_warp + cc) * p.stride_d + dr_warp + cr] = D_TYPE(sums[sums_idx].x);
305
+ }
306
+ #endif // MUL_MAT_ID
307
+ }
308
+ }
309
+ }
310
+ }
311
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl ADDED
@@ -0,0 +1,454 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
2
+ #extension GL_EXT_shader_explicit_arithmetic_types_int16 : require
3
+ #extension GL_EXT_shader_explicit_arithmetic_types_int8 : require
4
+
5
+ #include "types.glsl"
6
+
7
+ // Each iqs value maps to a 32-bit integer
8
+
9
+ #if defined(DATA_A_Q4_0) || defined(DATA_A_Q4_1)
10
+ // 2-byte loads for Q4_0 blocks (18 bytes)
11
+ // 4-byte loads for Q4_1 blocks (20 bytes)
12
+ void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) {
13
+ #ifdef DATA_A_Q4_0
14
+ buf_a[buf_ib].qs[iqs] = pack32(u16vec2(data_a_packed16[ib].qs[iqs * 2],
15
+ data_a_packed16[ib].qs[iqs * 2 + 1]));
16
+
17
+ if (iqs == 0) {
18
+ buf_a[buf_ib].dm = FLOAT_TYPE(data_a_packed16[ib].d);
19
+ }
20
+ #else // DATA_A_Q4_1
21
+ buf_a[buf_ib].qs[iqs] = data_a_packed32[ib].qs[iqs];
22
+
23
+ if (iqs == 0) {
24
+ buf_a[buf_ib].dm = FLOAT_TYPEV2(data_a_packed32[ib].dm);
25
+ }
26
+ #endif
27
+ }
28
+
29
+ void block_a_to_registers(const uint reg_ib, const uint buf_ib) {
30
+ cache_a[reg_ib].dm = buf_a[buf_ib].dm;
31
+
32
+ [[unroll]] for (uint iqs = 0; iqs < 4; iqs++) {
33
+ cache_a[reg_ib].qs[iqs] = buf_a[buf_ib].qs[iqs];
34
+ }
35
+ }
36
+
37
+ ACC_TYPE mmq_dot_product(const uint ib_a) {
38
+ int32_t q_sum = 0;
39
+ [[unroll]] for (uint iqs = 0; iqs < 4; iqs++) {
40
+ const uint32_t vui = cache_a[ib_a].qs[iqs];
41
+ const i32vec2 qs_a = i32vec2( vui & 0x0F0F0F0F,
42
+ (vui >> 4) & 0x0F0F0F0F);
43
+
44
+ const int32_t qs_b0 = cache_b.qs[iqs];
45
+ const int32_t qs_b1 = cache_b.qs[iqs + 4];
46
+
47
+ q_sum += dotPacked4x8EXT(qs_a.x, qs_b0);
48
+ q_sum += dotPacked4x8EXT(qs_a.y, qs_b1);
49
+ }
50
+
51
+ #ifdef DATA_A_Q4_0
52
+ return ACC_TYPE(float(cache_a[ib_a].dm) * (float(q_sum) * float(cache_b.ds.x) - 8.0 * float(cache_b.ds.y)));
53
+ #else // DATA_A_Q4_1
54
+ return ACC_TYPE(float(q_sum) * float(cache_a[ib_a].dm.x) * float(cache_b.ds.x) + float(cache_a[ib_a].dm.y) * float(cache_b.ds.y));
55
+ #endif
56
+ }
57
+ #endif
58
+
59
+ #if defined(DATA_A_Q5_0) || defined(DATA_A_Q5_1)
60
+ // 2-byte loads for Q5_0 blocks (22 bytes)
61
+ // 4-byte loads for Q5_1 blocks (24 bytes)
62
+ void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) {
63
+ #ifdef DATA_A_Q5_0
64
+ buf_a[buf_ib].qs[iqs] = pack32(u16vec2(data_a_packed16[ib].qs[iqs * 2],
65
+ data_a_packed16[ib].qs[iqs * 2 + 1]));
66
+
67
+ if (iqs == 0) {
68
+ buf_a[buf_ib].dm = FLOAT_TYPE(data_a_packed16[ib].d);
69
+ buf_a[buf_ib].qh = pack32(u16vec2(data_a_packed16[ib].qh[0], data_a_packed16[ib].qh[1]));
70
+ }
71
+ #else // DATA_A_Q5_1
72
+ buf_a[buf_ib].qs[iqs] = data_a_packed32[ib].qs[iqs];
73
+
74
+ if (iqs == 0) {
75
+ buf_a[buf_ib].dm = FLOAT_TYPEV2(data_a_packed32[ib].dm);
76
+ buf_a[buf_ib].qh = data_a_packed32[ib].qh;
77
+ }
78
+ #endif
79
+ }
80
+
81
+ void block_a_to_registers(const uint reg_ib, const uint buf_ib) {
82
+ cache_a[reg_ib].dm = buf_a[buf_ib].dm;
83
+ cache_a[reg_ib].qh = buf_a[buf_ib].qh;
84
+
85
+ [[unroll]] for (uint iqs = 0; iqs < 4; iqs++) {
86
+ cache_a[reg_ib].qs[iqs] = buf_a[buf_ib].qs[iqs];
87
+ }
88
+ }
89
+
90
+ ACC_TYPE mmq_dot_product(const uint ib_a) {
91
+ int32_t q_sum = 0;
92
+ [[unroll]] for (uint iqs = 0; iqs < 4; iqs++) {
93
+ const uint32_t vui = cache_a[ib_a].qs[iqs];
94
+ const int32_t qh = int32_t(cache_a[ib_a].qh >> (4 * iqs));
95
+ const int32_t qs_a0 = int32_t(vui & 0x0F0F0F0F)
96
+ | ((qh & 0xF) * 0x02040810) & 0x10101010; // (0,1,2,3) -> (4,12,20,28)
97
+ const int32_t qs_a1 = int32_t((vui >> 4) & 0x0F0F0F0F)
98
+ | (((qh >> 16) & 0xF) * 0x02040810) & 0x10101010; // (16,17,18,19) -> (4,12,20,28)
99
+
100
+ const int32_t qs_b0 = cache_b.qs[iqs];
101
+ const int32_t qs_b1 = cache_b.qs[iqs + 4];
102
+
103
+ q_sum += dotPacked4x8EXT(qs_a0, qs_b0);
104
+ q_sum += dotPacked4x8EXT(qs_a1, qs_b1);
105
+ }
106
+
107
+ #ifdef DATA_A_Q5_0
108
+ return ACC_TYPE(float(cache_a[ib_a].dm) * (float(q_sum) * float(cache_b.ds.x) - 16.0 * float(cache_b.ds.y)));
109
+ #else // DATA_A_Q5_1
110
+ return ACC_TYPE(float(q_sum) * float(cache_a[ib_a].dm.x) * float(cache_b.ds.x) + float(cache_a[ib_a].dm.y) * float(cache_b.ds.y));
111
+ #endif
112
+ }
113
+ #endif
114
+
115
+ #if defined(DATA_A_Q8_0)
116
+ // 2-byte loads for Q8_0 blocks (34 bytes)
117
+ void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) {
118
+ buf_a[buf_ib].qs[iqs] = pack32(i16vec2(data_a_packed16[ib].qs[iqs * 2],
119
+ data_a_packed16[ib].qs[iqs * 2 + 1]));
120
+
121
+ if (iqs == 0) {
122
+ buf_a[buf_ib].dm = FLOAT_TYPE(data_a_packed16[ib].d);
123
+ }
124
+ }
125
+
126
+ void block_a_to_registers(const uint reg_ib, const uint buf_ib) {
127
+ cache_a[reg_ib].dm = buf_a[buf_ib].dm;
128
+
129
+ [[unroll]] for (uint iqs = 0; iqs < 8; iqs++) {
130
+ cache_a[reg_ib].qs[iqs] = buf_a[buf_ib].qs[iqs];
131
+ }
132
+ }
133
+
134
+ ACC_TYPE mmq_dot_product(const uint ib_a) {
135
+ int32_t q_sum = 0;
136
+ [[unroll]] for (uint iqs = 0; iqs < 8; iqs++) {
137
+ const int32_t qs_a = cache_a[ib_a].qs[iqs];
138
+ const int32_t qs_b = cache_b.qs[iqs];
139
+
140
+ q_sum += dotPacked4x8EXT(qs_a, qs_b);
141
+ }
142
+
143
+ return ACC_TYPE(float(q_sum) * float(cache_a[ib_a].dm) * float(cache_b.ds.x));
144
+ }
145
+ #endif
146
+
147
+ #if defined(DATA_A_MXFP4)
148
+ // 1-byte loads for mxfp4 blocks (17 bytes)
149
+ void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) {
150
+ const uint32_t qs = pack32(u8vec4(data_a[ib].qs[iqs * 4 ],
151
+ data_a[ib].qs[iqs * 4 + 1],
152
+ data_a[ib].qs[iqs * 4 + 2],
153
+ data_a[ib].qs[iqs * 4 + 3]));
154
+
155
+ const u8vec4 i_a0 = unpack8( qs & 0x0F0F0F0F);
156
+ const u8vec4 i_a1 = unpack8((qs >> 4) & 0x0F0F0F0F);
157
+
158
+ buf_a[buf_ib].qs[iqs ] = pack32(i8vec4(kvalues_mxfp4[i_a0.x], kvalues_mxfp4[i_a0.y], kvalues_mxfp4[i_a0.z], kvalues_mxfp4[i_a0.w]));
159
+ buf_a[buf_ib].qs[iqs + 4] = pack32(i8vec4(kvalues_mxfp4[i_a1.x], kvalues_mxfp4[i_a1.y], kvalues_mxfp4[i_a1.z], kvalues_mxfp4[i_a1.w]));
160
+
161
+ if (iqs == 0) {
162
+ buf_a[buf_ib].d = FLOAT_TYPE(e8m0_to_fp32(data_a[ib].e) * 0.5);
163
+ }
164
+ }
165
+
166
+ void block_a_to_registers(const uint reg_ib, const uint buf_ib) {
167
+ cache_a[reg_ib].d = buf_a[buf_ib].d;
168
+
169
+ [[unroll]] for (uint iqs = 0; iqs < 8; iqs++) {
170
+ cache_a[reg_ib].qs[iqs] = buf_a[buf_ib].qs[iqs];
171
+ }
172
+ }
173
+
174
+ ACC_TYPE mmq_dot_product(const uint ib_a) {
175
+ int32_t q_sum = 0;
176
+ [[unroll]] for (uint iqs = 0; iqs < 8; iqs++) {
177
+ const int32_t qs_a = cache_a[ib_a].qs[iqs];
178
+
179
+ q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
180
+ }
181
+
182
+ return ACC_TYPE(float(cache_a[ib_a].d) * float(cache_b.ds.x) * float(q_sum));
183
+ }
184
+ #endif
185
+
186
+ // For k-quants, ib and iqs still assume 32-wide blocks, but k-quants are 256-wide
187
+ // iqs still refers to a 32-bit integer, meaning 0..7 for 32-wide quants
188
+ #if defined(DATA_A_Q2_K)
189
+ // 4-byte loads for Q2_K blocks (84 bytes)
190
+ void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) {
191
+ const uint ib_k = ib / 8;
192
+ const uint iqs_k = (ib % 8) * 8 + iqs * QUANT_R_MMQ;
193
+
194
+ const uint qs_idx = (iqs_k / 32) * 8 + (iqs_k % 8);
195
+ const uint qs_shift = ((iqs_k % 32) / 8) * 2;
196
+
197
+ // Repack 4x4 quants into one int
198
+ const uint32_t vals0 = (data_a_packed32[ib_k].qs[qs_idx ] >> qs_shift) & 0x03030303;
199
+ const uint32_t vals1 = (data_a_packed32[ib_k].qs[qs_idx + 1] >> qs_shift) & 0x03030303;
200
+ const uint32_t vals2 = (data_a_packed32[ib_k].qs[qs_idx + 2] >> qs_shift) & 0x03030303;
201
+ const uint32_t vals3 = (data_a_packed32[ib_k].qs[qs_idx + 3] >> qs_shift) & 0x03030303;
202
+
203
+ buf_a[buf_ib].qs[iqs] = vals0 | (vals1 << 2) | (vals2 << 4) | (vals3 << 6);
204
+
205
+ if (iqs == 0) {
206
+ buf_a[buf_ib].dm = FLOAT_TYPEV2(data_a_packed32[ib_k].dm);
207
+ buf_a[buf_ib].scales = unpack8(uint32_t(data_a_packed16[ib_k].scales[iqs_k / 8])).xy; // vec4 used due to #12147
208
+ }
209
+ }
210
+
211
+ void block_a_to_registers(const uint reg_ib, const uint buf_ib) {
212
+ cache_a[reg_ib].dm = buf_a[buf_ib].dm;
213
+ cache_a[reg_ib].scales = buf_a[buf_ib].scales;
214
+
215
+ [[unroll]] for (uint iqs = 0; iqs < 2; iqs++) {
216
+ cache_a[reg_ib].qs[iqs] = buf_a[buf_ib].qs[iqs];
217
+ }
218
+ }
219
+
220
+ ACC_TYPE mmq_dot_product(const uint ib_a) {
221
+ int32_t sum_d = 0;
222
+ int32_t sum_m = 0;
223
+
224
+ [[unroll]] for (uint iqs = 0; iqs < 8; iqs++) {
225
+ const uint8_t scale = cache_a[ib_a].scales[iqs / 4];
226
+ const int32_t scale_m = int32_t(scale >> 4) * 0x01010101; // Duplicate 8-bit value across 32-bits.
227
+ const int32_t qs_a = int32_t((cache_a[ib_a].qs[iqs / 4] >> ((iqs % 4) * 2)) & 0x03030303);
228
+
229
+ sum_d += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]) * (scale & 0xF);
230
+ sum_m += dotPacked4x8EXT(scale_m, cache_b.qs[iqs]);
231
+ }
232
+
233
+ return ACC_TYPE(float(cache_b.ds.x) * (float(cache_a[ib_a].dm.x) * float(sum_d) - float(cache_a[ib_a].dm.y) * float(sum_m)));
234
+ }
235
+ #endif
236
+
237
+ #if defined(DATA_A_Q3_K)
238
+ // 2-byte loads for Q3_K blocks (110 bytes)
239
+ void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) {
240
+ const uint ib_k = ib / 8;
241
+ const uint hm_idx = iqs * QUANT_R_MMQ;
242
+ const uint iqs_k = (ib % 8) * 8 + hm_idx;
243
+
244
+ const uint qs_idx = (iqs_k / 32) * 8 + (iqs_k % 8);
245
+ const uint qs_shift = ((iqs_k % 32) / 8) * 2;
246
+ const uint hm_shift = iqs_k / 8;
247
+
248
+ // Repack 2x4 quants into one int
249
+ // Add the 3rd bit instead of subtracting it to allow packing the quants
250
+ // vec4 for unpack8 used due to #12147
251
+ const i8vec2 vals00 = unpack8(int32_t(int16_t((data_a_packed16[ib_k].qs[qs_idx * 2 ] >> qs_shift) & uint16_t(0x0303)))).xy |
252
+ unpack8(int32_t(int16_t(((data_a_packed16[ib_k].hmask[hm_idx * 2 ] >> hm_shift) & uint16_t(0x0101))) << 2)).xy;
253
+ const i8vec2 vals01 = unpack8(int32_t(int16_t((data_a_packed16[ib_k].qs[qs_idx * 2 + 1 ] >> qs_shift) & uint16_t(0x0303)))).xy |
254
+ unpack8(int32_t(int16_t(((data_a_packed16[ib_k].hmask[hm_idx * 2 + 1] >> hm_shift) & uint16_t(0x0101))) << 2)).xy;
255
+ const i8vec2 vals10 = unpack8(int32_t(int16_t((data_a_packed16[ib_k].qs[qs_idx * 2 + 2 ] >> qs_shift) & uint16_t(0x0303)))).xy |
256
+ unpack8(int32_t(int16_t(((data_a_packed16[ib_k].hmask[hm_idx * 2 + 2] >> hm_shift) & uint16_t(0x0101))) << 2)).xy;
257
+ const i8vec2 vals11 = unpack8(int32_t(int16_t((data_a_packed16[ib_k].qs[qs_idx * 2 + 3 ] >> qs_shift) & uint16_t(0x0303)))).xy |
258
+ unpack8(int32_t(int16_t(((data_a_packed16[ib_k].hmask[hm_idx * 2 + 3] >> hm_shift) & uint16_t(0x0101))) << 2)).xy;
259
+ buf_a[buf_ib].qs[iqs] = pack32(u8vec4(vals00.x, vals00.y, vals01.x, vals01.y)) |
260
+ (pack32(u8vec4(vals10.x, vals10.y, vals11.x, vals11.y)) << 4);
261
+
262
+ if (iqs == 0) {
263
+ const uint is = iqs_k / 4;
264
+ const i8vec2 scales = i8vec2(unpack8(uint32_t(((data_a_packed16[ib_k].scales[(is % 8 ) / 2] >> (4 * (is / 8))) & 0x0F0F) |
265
+ (((data_a_packed16[ib_k].scales[(8 + (is % 4)) / 2] >> (2 * (is / 4))) & 0x0303) << 4))).xy); // vec4 used due to #12147
266
+
267
+ buf_a[buf_ib].d_scales = FLOAT_TYPEV2(float(data_a_packed16[ib_k].d) * vec2(scales - 32));
268
+ }
269
+ }
270
+
271
+ void block_a_to_registers(const uint reg_ib, const uint buf_ib) {
272
+ cache_a[reg_ib].d_scales = buf_a[buf_ib].d_scales;
273
+
274
+ [[unroll]] for (uint iqs = 0; iqs < 4; iqs++) {
275
+ cache_a[reg_ib].qs[iqs] = buf_a[buf_ib].qs[iqs];
276
+ }
277
+ }
278
+
279
+ ACC_TYPE mmq_dot_product(const uint ib_a) {
280
+ float result = 0.0;
281
+ int32_t q_sum = 0;
282
+
283
+ [[unroll]] for (uint iqs = 0; iqs < 4; iqs++) {
284
+ // Subtract 4 from the quants to correct the 3rd bit offset
285
+ const int32_t qs_a = pack32(unpack8(int32_t((cache_a[ib_a].qs[iqs / 2] >> ((iqs % 2) * 4)) & 0x0F0F0F0F)) - int8_t(4));
286
+
287
+ q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
288
+ }
289
+ result += float(cache_a[ib_a].d_scales[0]) * float(q_sum);
290
+ q_sum = 0;
291
+
292
+ [[unroll]] for (uint iqs = 4; iqs < 8; iqs++) {
293
+ const int32_t qs_a = pack32(unpack8(int32_t((cache_a[ib_a].qs[iqs / 2] >> ((iqs % 2) * 4)) & 0x0F0F0F0F)) - int8_t(4));
294
+
295
+ q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
296
+ }
297
+ result += float(cache_a[ib_a].d_scales[1]) * float(q_sum);
298
+
299
+ return ACC_TYPE(float(cache_b.ds.x) * result);
300
+ }
301
+ #endif
302
+
303
+ #if defined(DATA_A_Q4_K) || defined(DATA_A_Q5_K)
304
+ // 4-byte loads for Q4_K blocks (144 bytes) and Q5_K blocks (176 bytes)
305
+ void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) {
306
+ const uint ib_k = ib / 8;
307
+ const uint iqs_k = (ib % 8) * 8 + iqs * QUANT_R_MMQ;
308
+
309
+ const uint qs_idx = (iqs_k / 16) * 8 + (iqs_k % 8);
310
+ const uint qs_shift = ((iqs_k % 16) / 8) * 4;
311
+
312
+ // Repack 2x4 quants into one int
313
+ #if defined(DATA_A_Q4_K)
314
+ const uint32_t vals0 = (data_a_packed32[ib_k].qs[qs_idx ] >> qs_shift) & 0x0F0F0F0F;
315
+ const uint32_t vals1 = (data_a_packed32[ib_k].qs[qs_idx + 1] >> qs_shift) & 0x0F0F0F0F;
316
+
317
+ buf_a[buf_ib].qs[iqs] = vals0 | (vals1 << 4);
318
+ #else // defined(DATA_A_Q5_K)
319
+ const uint qh_idx = iqs * QUANT_R_MMQ;
320
+ const uint qh_shift = iqs_k / 8;
321
+
322
+ buf_a[buf_ib].qs[iqs] = int32_t(((data_a_packed32[ib_k].qs[qs_idx] >> qs_shift) & 0x0F0F0F0F) |
323
+ (((data_a_packed32[ib_k].qh[qh_idx] >> qh_shift) & 0x01010101) << 4));
324
+ #endif
325
+
326
+ if (iqs == 0) {
327
+ // Scale index
328
+ const uint is = iqs_k / 8;
329
+ u8vec2 scale_dm;
330
+ if (is < 4) {
331
+ scale_dm = u8vec2(data_a[ib_k].scales[is] & 0x3F, data_a[ib_k].scales[is + 4] & 0x3F);
332
+ } else {
333
+ scale_dm = u8vec2((data_a[ib_k].scales[is+4] & 0xF) | ((data_a[ib_k].scales[is-4] & 0xC0) >> 2),
334
+ (data_a[ib_k].scales[is+4] >> 4) | ((data_a[ib_k].scales[is ] & 0xC0) >> 2));
335
+ }
336
+
337
+ buf_a[buf_ib].dm = FLOAT_TYPEV2(vec2(data_a_packed32[ib_k].dm) * vec2(scale_dm));
338
+ }
339
+ }
340
+
341
+ void block_a_to_registers(const uint reg_ib, const uint buf_ib) {
342
+ cache_a[reg_ib].dm = buf_a[buf_ib].dm;
343
+
344
+ [[unroll]] for (uint iqs = 0; iqs < 8 / QUANT_R_MMQ; iqs++) {
345
+ cache_a[reg_ib].qs[iqs] = buf_a[buf_ib].qs[iqs];
346
+ }
347
+ }
348
+
349
+ ACC_TYPE mmq_dot_product(const uint ib_a) {
350
+ int32_t q_sum = 0;
351
+
352
+ [[unroll]] for (uint iqs = 0; iqs < 8; iqs++) {
353
+ #if defined(DATA_A_Q4_K)
354
+ const int32_t qs_a = int32_t((cache_a[ib_a].qs[iqs / 2] >> ((iqs % 2) * 4)) & 0x0F0F0F0F);
355
+ #else // defined(DATA_A_Q5_K)
356
+ const int32_t qs_a = cache_a[ib_a].qs[iqs];
357
+ #endif
358
+
359
+ q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
360
+ }
361
+
362
+ return ACC_TYPE(float(cache_b.ds.x) * float(cache_a[ib_a].dm.x) * float(q_sum) - float(cache_a[ib_a].dm.y) * float(cache_b.ds.y));
363
+ }
364
+ #endif
365
+
366
+ #if defined(DATA_A_Q6_K)
367
+ // 2-byte loads for Q6_K blocks (210 bytes)
368
+ void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) {
369
+ const uint ib_k = ib / 8;
370
+ const uint iqs_k = (ib % 8) * 8 + iqs;
371
+
372
+ const uint ql_idx = (iqs_k / 32) * 16 + iqs_k % 16;
373
+ const uint ql_shift = ((iqs_k % 32) / 16) * 4;
374
+
375
+ const uint qh_idx = (iqs_k / 32) * 8 + iqs;
376
+ const uint qh_shift = ((iqs_k % 32) / 8) * 2;
377
+
378
+ const i8vec2 vals00 = (unpack8(int32_t((data_a_packed16[ib_k].ql[ql_idx * 2 ] >> ql_shift) & uint16_t(0x0F0F))).xy |
379
+ unpack8(int32_t(((data_a_packed16[ib_k].qh[qh_idx * 2 ] >> qh_shift) & uint16_t(0x0303)) << 4)).xy) - int8_t(32);
380
+ const i8vec2 vals01 = (unpack8(int32_t((data_a_packed16[ib_k].ql[ql_idx * 2 + 1] >> ql_shift) & uint16_t(0x0F0F))).xy |
381
+ unpack8(int32_t(((data_a_packed16[ib_k].qh[qh_idx * 2 + 1] >> qh_shift) & uint16_t(0x0303)) << 4)).xy) - int8_t(32);
382
+ buf_a[buf_ib].qs[iqs] = pack32(i8vec4(vals00.x, vals00.y, vals01.x, vals01.y));
383
+
384
+ if (iqs == 0) {
385
+ const uint is = iqs_k / 4;
386
+ const i8vec2 scales = unpack8(int32_t(data_a_packed16[ib_k].scales[is / 2])).xy;
387
+
388
+ buf_a[buf_ib].d_scales = FLOAT_TYPEV2(float(data_a_packed16[ib_k].d) * vec2(scales));
389
+ }
390
+ }
391
+
392
+ void block_a_to_registers(const uint reg_ib, const uint buf_ib) {
393
+ cache_a[reg_ib].d_scales = buf_a[buf_ib].d_scales;
394
+
395
+ [[unroll]] for (uint iqs = 0; iqs < 8; iqs++) {
396
+ cache_a[reg_ib].qs[iqs] = buf_a[buf_ib].qs[iqs];
397
+ }
398
+ }
399
+
400
+ ACC_TYPE mmq_dot_product(const uint ib_a) {
401
+ float result = 0.0;
402
+ int32_t q_sum = 0;
403
+
404
+ [[unroll]] for (uint iqs = 0; iqs < 4; iqs++) {
405
+ const int32_t qs_a = cache_a[ib_a].qs[iqs];
406
+
407
+ q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
408
+ }
409
+ result += float(cache_a[ib_a].d_scales[0]) * float(q_sum);
410
+ q_sum = 0;
411
+
412
+ [[unroll]] for (uint iqs = 4; iqs < 8; iqs++) {
413
+ const int32_t qs_a = cache_a[ib_a].qs[iqs];
414
+
415
+ q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
416
+ }
417
+ result += float(cache_a[ib_a].d_scales[1]) * float(q_sum);
418
+
419
+ return ACC_TYPE(float(cache_b.ds.x) * result);
420
+ }
421
+ #endif
422
+
423
+ void block_b_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bool is_in_bounds) {
424
+ if (is_in_bounds) {
425
+ const uint ib_outer = ib / 4;
426
+ const uint ib_inner = ib % 4;
427
+
428
+ if (iqs == 0) {
429
+ buf_b[buf_ib].ds = FLOAT_TYPEV2(data_b[ib_outer].ds[ib_inner]);
430
+ }
431
+
432
+ const ivec4 values = data_b[ib_outer].qs[ib_inner * 2 + iqs];
433
+ buf_b[buf_ib].qs[iqs * 4 ] = values.x;
434
+ buf_b[buf_ib].qs[iqs * 4 + 1] = values.y;
435
+ buf_b[buf_ib].qs[iqs * 4 + 2] = values.z;
436
+ buf_b[buf_ib].qs[iqs * 4 + 3] = values.w;
437
+ } else {
438
+ if (iqs == 0) {
439
+ buf_b[buf_ib].ds = FLOAT_TYPEV2(0.0f);
440
+ }
441
+
442
+ buf_b[buf_ib].qs[iqs * 4 ] = 0;
443
+ buf_b[buf_ib].qs[iqs * 4 + 1] = 0;
444
+ buf_b[buf_ib].qs[iqs * 4 + 2] = 0;
445
+ buf_b[buf_ib].qs[iqs * 4 + 3] = 0;
446
+ }
447
+ }
448
+
449
+ void block_b_to_registers(const uint ib) {
450
+ cache_b.ds = buf_b[ib].ds;
451
+ [[unroll]] for (uint iqs = 0; iqs < BK / 4; iqs++) {
452
+ cache_b.qs[iqs] = buf_b[ib].qs[iqs];
453
+ }
454
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl ADDED
@@ -0,0 +1,93 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #if defined(FA_MMQ_MIXED)
2
+ // Mixed-K flash attention MMQ: superset cache that fits Q4_0/Q4_1/Q5_0/Q5_1/Q8_0.
3
+ // Q4_*/Q5_* only use qs[0..3] and (for Q5_*) qh. Q8_0 uses qs[0..7]. Single-scale
4
+ // types (Q4_0/Q5_0/Q8_0) leave dm.y unused.
5
+ struct block_a_cache {
6
+ int32_t qs[8];
7
+ uint32_t qh;
8
+ FLOAT_TYPEV2 dm;
9
+ };
10
+ #elif defined(DATA_A_Q4_0)
11
+ #define QUANT_R_MMQ 2
12
+ struct block_a_cache {
13
+ uint32_t qs[16/4];
14
+ FLOAT_TYPE dm;
15
+ };
16
+ #elif defined(DATA_A_Q4_1)
17
+ #define QUANT_R_MMQ 2
18
+ struct block_a_cache {
19
+ uint32_t qs[16/4];
20
+ FLOAT_TYPEV2 dm;
21
+ };
22
+ #elif defined(DATA_A_Q5_0)
23
+ #define QUANT_R_MMQ 2
24
+ struct block_a_cache {
25
+ uint32_t qs[16/4];
26
+ uint32_t qh;
27
+ FLOAT_TYPE dm;
28
+ };
29
+ #elif defined(DATA_A_Q5_1)
30
+ #define QUANT_R_MMQ 2
31
+ struct block_a_cache {
32
+ uint32_t qs[16/4];
33
+ uint32_t qh;
34
+ FLOAT_TYPEV2 dm;
35
+ };
36
+ #elif defined(DATA_A_Q8_0)
37
+ #define QUANT_R_MMQ 1
38
+ // AMD likes 4, Intel likes 1 and Nvidia likes 2
39
+ // #define BK_STEP 1
40
+ struct block_a_cache {
41
+ int32_t qs[32/4];
42
+ FLOAT_TYPE dm;
43
+ };
44
+ #elif defined(DATA_A_IQ4_NL)
45
+ #define QUANT_R_MMQ 2
46
+ struct block_a_cache {
47
+ int32_t qs[8];
48
+ FLOAT_TYPE dm;
49
+ };
50
+ #elif defined(DATA_A_MXFP4)
51
+ #define QUANT_R_MMQ 2
52
+ struct block_a_cache {
53
+ int32_t qs[8];
54
+ FLOAT_TYPE d;
55
+ };
56
+ #elif defined(DATA_A_Q2_K)
57
+ #define QUANT_R_MMQ 4
58
+ struct block_a_cache {
59
+ uint32_t qs[2];
60
+ u8vec2 scales;
61
+ FLOAT_TYPEV2 dm;
62
+ };
63
+ #elif defined(DATA_A_Q3_K)
64
+ #define QUANT_R_MMQ 2
65
+ struct block_a_cache {
66
+ uint32_t qs[4];
67
+ FLOAT_TYPEV2 d_scales;
68
+ };
69
+ #elif defined(DATA_A_Q4_K)
70
+ #define QUANT_R_MMQ 2
71
+ struct block_a_cache {
72
+ uint32_t qs[4];
73
+ FLOAT_TYPEV2 dm;
74
+ };
75
+ #elif defined(DATA_A_Q5_K)
76
+ #define QUANT_R_MMQ 1
77
+ struct block_a_cache {
78
+ int32_t qs[8];
79
+ FLOAT_TYPEV2 dm;
80
+ };
81
+ #elif defined(DATA_A_Q6_K)
82
+ #define QUANT_R_MMQ 1
83
+ struct block_a_cache {
84
+ int32_t qs[8];
85
+ FLOAT_TYPEV2 d_scales;
86
+ };
87
+ #endif
88
+
89
+ struct block_b_cache
90
+ {
91
+ int32_t qs[8];
92
+ FLOAT_TYPEV2 ds;
93
+ };
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/multi_add.comp ADDED
@@ -0,0 +1,194 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+
3
+ #extension GL_EXT_shader_16bit_storage : require
4
+ #extension GL_EXT_nonuniform_qualifier : enable
5
+ #extension GL_EXT_control_flow_attributes : require
6
+ #if ADD_RMS
7
+ #extension GL_KHR_shader_subgroup_arithmetic : enable
8
+ #extension GL_KHR_shader_subgroup_basic : enable
9
+ #endif
10
+
11
+ #include "types.glsl"
12
+ #include "utils.glsl"
13
+
14
+ layout (push_constant) uniform parameter2
15
+ {
16
+ // shape for dst
17
+ uint ne20; uint ne21; uint ne22; uint ne23;
18
+
19
+ // strides for srcs+dst
20
+ uint nb[12][4];
21
+
22
+ uint rms_partials;
23
+ } p;
24
+
25
+ // No readonly/writeonly decorations. Workaround for MoltenVK Bug, see https://github.com/ggml-org/llama.cpp/issues/15498
26
+ layout (binding = 0) buffer A0 {A_TYPE data_a[];} a0;
27
+ layout (binding = 1) buffer A1 {A_TYPE data_a[];} a1;
28
+ layout (binding = 2) buffer A2 {A_TYPE data_a[];} a2;
29
+ layout (binding = 3) buffer A3 {A_TYPE data_a[];} a3;
30
+ layout (binding = 4) buffer A4 {A_TYPE data_a[];} a4;
31
+ layout (binding = 5) buffer A5 {A_TYPE data_a[];} a5;
32
+ layout (binding = 6) buffer A6 {A_TYPE data_a[];} a6;
33
+ layout (binding = 7) buffer A7 {A_TYPE data_a[];} a7;
34
+ layout (binding = 8) buffer A8 {A_TYPE data_a[];} a8;
35
+ layout (binding = 9) buffer A9 {A_TYPE data_a[];} a9;
36
+ layout (binding = 10) buffer A10 {A_TYPE data_a[];} a10;
37
+ layout (binding = 11) buffer A11 {A_TYPE data_a[];} a11;
38
+ layout (binding = 0) buffer D0 {D_TYPE data_d[];} d0;
39
+ layout (binding = 1) buffer D1 {D_TYPE data_d[];} d1;
40
+ layout (binding = 2) buffer D2 {D_TYPE data_d[];} d2;
41
+ layout (binding = 3) buffer D3 {D_TYPE data_d[];} d3;
42
+ layout (binding = 4) buffer D4 {D_TYPE data_d[];} d4;
43
+ layout (binding = 5) buffer D5 {D_TYPE data_d[];} d5;
44
+ layout (binding = 6) buffer D6 {D_TYPE data_d[];} d6;
45
+ layout (binding = 7) buffer D7 {D_TYPE data_d[];} d7;
46
+ layout (binding = 8) buffer D8 {D_TYPE data_d[];} d8;
47
+ layout (binding = 9) buffer D9 {D_TYPE data_d[];} d9;
48
+ layout (binding = 10) buffer D10 {D_TYPE data_d[];} d10;
49
+ layout (binding = 11) buffer D11 {D_TYPE data_d[];} d11;
50
+ layout (binding = 0, std430) buffer PartialBuf0 {float partial_sums[];} partials0;
51
+ layout (binding = 1, std430) buffer PartialBuf1 {float partial_sums[];} partials1;
52
+ layout (binding = 2, std430) buffer PartialBuf2 {float partial_sums[];} partials2;
53
+ layout (binding = 3, std430) buffer PartialBuf3 {float partial_sums[];} partials3;
54
+ layout (binding = 4, std430) buffer PartialBuf4 {float partial_sums[];} partials4;
55
+ layout (binding = 5, std430) buffer PartialBuf5 {float partial_sums[];} partials5;
56
+ layout (binding = 6, std430) buffer PartialBuf6 {float partial_sums[];} partials6;
57
+ layout (binding = 7, std430) buffer PartialBuf7 {float partial_sums[];} partials7;
58
+ layout (binding = 8, std430) buffer PartialBuf8 {float partial_sums[];} partials8;
59
+ layout (binding = 9, std430) buffer PartialBuf9 {float partial_sums[];} partials9;
60
+ layout (binding = 10, std430) buffer PartialBuf10 {float partial_sums[];} partials10;
61
+ layout (binding = 11, std430) buffer PartialBuf11 {float partial_sums[];} partials11;
62
+
63
+ layout(constant_id = 0) const uint num_srcs = 2;
64
+
65
+ FLOAT_TYPE load_a(uint b, uint i) {
66
+ switch (b) {
67
+ case 0: return FLOAT_TYPE(a0.data_a[i]);
68
+ case 1: return FLOAT_TYPE(a1.data_a[i]);
69
+ case 2: return FLOAT_TYPE(a2.data_a[i]);
70
+ case 3: return FLOAT_TYPE(a3.data_a[i]);
71
+ case 4: return FLOAT_TYPE(a4.data_a[i]);
72
+ case 5: return FLOAT_TYPE(a5.data_a[i]);
73
+ case 6: return FLOAT_TYPE(a6.data_a[i]);
74
+ case 7: return FLOAT_TYPE(a7.data_a[i]);
75
+ case 8: return FLOAT_TYPE(a8.data_a[i]);
76
+ case 9: return FLOAT_TYPE(a9.data_a[i]);
77
+ case 10: return FLOAT_TYPE(a10.data_a[i]);
78
+ case 11: return FLOAT_TYPE(a11.data_a[i]);
79
+ default: return FLOAT_TYPE(0);
80
+ }
81
+ }
82
+
83
+ void store_d(uint b, uint i, FLOAT_TYPE v) {
84
+ switch (b) {
85
+ case 0: d0.data_d[i] = D_TYPE(v); break;
86
+ case 1: d1.data_d[i] = D_TYPE(v); break;
87
+ case 2: d2.data_d[i] = D_TYPE(v); break;
88
+ case 3: d3.data_d[i] = D_TYPE(v); break;
89
+ case 4: d4.data_d[i] = D_TYPE(v); break;
90
+ case 5: d5.data_d[i] = D_TYPE(v); break;
91
+ case 6: d6.data_d[i] = D_TYPE(v); break;
92
+ case 7: d7.data_d[i] = D_TYPE(v); break;
93
+ case 8: d8.data_d[i] = D_TYPE(v); break;
94
+ case 9: d9.data_d[i] = D_TYPE(v); break;
95
+ case 10: d10.data_d[i] = D_TYPE(v); break;
96
+ case 11: d11.data_d[i] = D_TYPE(v); break;
97
+ default: break;
98
+ }
99
+ }
100
+
101
+ void store_partial(uint b, uint i, float v) {
102
+ switch (b) {
103
+ case 0: partials0.partial_sums[i] = v; break;
104
+ case 1: partials1.partial_sums[i] = v; break;
105
+ case 2: partials2.partial_sums[i] = v; break;
106
+ case 3: partials3.partial_sums[i] = v; break;
107
+ case 4: partials4.partial_sums[i] = v; break;
108
+ case 5: partials5.partial_sums[i] = v; break;
109
+ case 6: partials6.partial_sums[i] = v; break;
110
+ case 7: partials7.partial_sums[i] = v; break;
111
+ case 8: partials8.partial_sums[i] = v; break;
112
+ case 9: partials9.partial_sums[i] = v; break;
113
+ case 10: partials10.partial_sums[i] = v; break;
114
+ case 11: partials11.partial_sums[i] = v; break;
115
+ default: break;
116
+ }
117
+ }
118
+
119
+ uint src_idx(uint s, uint i00, uint i01, uint i02, uint i03) {
120
+ return i03*p.nb[s][3] + i02*p.nb[s][2] + i01*p.nb[s][1] + i00*p.nb[s][0];
121
+ }
122
+
123
+ uint dst_idx(uint i00, uint i01, uint i02, uint i03) {
124
+ uint nb20 = p.nb[num_srcs][0];
125
+ uint nb21 = p.nb[num_srcs][1];
126
+ uint nb22 = p.nb[num_srcs][2];
127
+ uint nb23 = p.nb[num_srcs][3];
128
+ return i03*nb23 + i02*nb22 + i01*nb21 + i00*nb20;
129
+ }
130
+
131
+ uint get_idx() {
132
+ return gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;
133
+ }
134
+
135
+ const uint num_threads = 256;
136
+
137
+ layout(local_size_x = num_threads, local_size_y = 1, local_size_z = 1) in;
138
+
139
+ #if ADD_RMS
140
+ // XXX TODO this could be sized based on number of subgroups, but that't not considered a constant
141
+ shared FLOAT_TYPE sumsh[num_threads];
142
+ #endif
143
+
144
+ void main() {
145
+ uint idx = get_idx();
146
+ uint orig_idx = idx;
147
+
148
+ uint ne = p.ne20 * p.ne21 * p.ne22 * p.ne23;
149
+
150
+ // num_threads * num_iter must equal 512, to match the wg_denoms and get_idx calculation
151
+ const uint num_iter = 2;
152
+
153
+ FLOAT_TYPE sum_sq = 0;
154
+
155
+ [[unroll]] for (uint i = 0; i < num_iter; ++i) {
156
+ if (idx >= ne) {
157
+ continue;
158
+ }
159
+ uint i00, i01, i02, i03;
160
+ get_indices(idx, i00, i01, i02, i03, p.ne20, p.ne21, p.ne22, p.ne23);
161
+
162
+ FLOAT_TYPE sum = FLOAT_TYPE(0);
163
+ [[unroll]] for (uint s = 0; s < num_srcs; ++s) {
164
+ sum += load_a(s, src_idx(s, i00, i01, i02, i03));
165
+ }
166
+ sum_sq += sum*sum;
167
+ store_d(num_srcs, dst_idx(i00, i01, i02, i03), sum);
168
+
169
+ idx += num_threads;
170
+ }
171
+
172
+ #if ADD_RMS
173
+ if (p.rms_partials != 0) {
174
+ // reduce the sum within each subgroup, then across subgroups
175
+ const uint NumSubgroups = num_threads / gl_SubgroupSize;
176
+ sum_sq = subgroupAdd(sum_sq);
177
+ if (gl_SubgroupInvocationID == 0) {
178
+ sumsh[gl_SubgroupID] = sum_sq;
179
+ }
180
+ barrier();
181
+ [[unroll]] for (uint s = NumSubgroups / 2; s > 0; s >>= 1) {
182
+ if (gl_SubgroupID < s && gl_SubgroupInvocationID == 0) {
183
+ sum_sq += sumsh[gl_SubgroupID + s];
184
+ sumsh[gl_SubgroupID] = sum_sq;
185
+ }
186
+ barrier();
187
+ }
188
+
189
+ if (gl_SubgroupID == 0 && gl_SubgroupInvocationID == 0) {
190
+ store_partial(num_srcs + 1, orig_idx / (num_iter * num_threads), sum_sq);
191
+ }
192
+ }
193
+ #endif
194
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+
3
+ #include "types.glsl"
4
+ #include "generic_unary_head.glsl"
5
+
6
+ #extension GL_EXT_control_flow_attributes : enable
7
+ #define BLOCK_SIZE 512
8
+
9
+ layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in;
10
+
11
+ shared vec2 sum[BLOCK_SIZE];
12
+
13
+ void main() {
14
+ const uint row = gl_WorkGroupID.z * 262144 + gl_WorkGroupID.y * 512 + gl_WorkGroupID.x;
15
+ const uint tid = gl_LocalInvocationID.x;
16
+
17
+ const uint a_base = get_aoffset() + src0_idx(row * p.ne00);
18
+ const uint d_base = get_doffset() + dst_idx(row * p.ne10);
19
+
20
+ sum[tid] = vec2(0.0f, 0.0f);
21
+
22
+ [[unroll]] for (uint i0 = tid; i0 < p.ne00; i0 += BLOCK_SIZE) {
23
+ const float xi = float(data_a[a_base + i0*p.nb00]);
24
+ sum[tid].x += xi;
25
+ sum[tid].y += xi * xi;
26
+ }
27
+
28
+ // sum up partial sums and write back result
29
+ barrier();
30
+ [[unroll]] for (int s = BLOCK_SIZE / 2; s > 0; s >>= 1) {
31
+ if (tid < s) {
32
+ sum[tid] += sum[tid + s];
33
+ }
34
+ barrier();
35
+ }
36
+
37
+ const float mean = sum[0].x / p.ne00;
38
+ const float var = sum[0].y / p.ne00 - mean * mean;
39
+ const float inv_std = inversesqrt(var + p.param1);
40
+
41
+ [[unroll]] for (uint i0 = tid; i0 < p.ne00; i0 += BLOCK_SIZE) {
42
+ data_d[d_base + i0*p.nb10] = D_TYPE((float(data_a[a_base + i0*p.nb00]) - mean) * inv_std);
43
+ }
44
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+
3
+ #include "generic_head.glsl"
4
+ #include "types.glsl"
5
+
6
+ #extension GL_EXT_control_flow_attributes : enable
7
+
8
+ layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
9
+
10
+ layout (binding = 0) buffer X {A_TYPE x[];};
11
+ layout (binding = 1) readonly buffer G {A_TYPE grad[];};
12
+ layout (binding = 2) buffer GM {A_TYPE gradm[];};
13
+ layout (binding = 3) buffer GV {A_TYPE gradv[];};
14
+ layout (binding = 4) readonly buffer P {float params[7];};
15
+
16
+ void main() {
17
+ const uint i = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;
18
+
19
+ if (i >= p.KX) {
20
+ return;
21
+ }
22
+
23
+ const float alpha = params[0];
24
+ const float beta1 = params[1];
25
+ const float beta2 = params[2];
26
+ const float eps = params[3];
27
+ const float wd = params[4];
28
+ const float beta1h = params[5];
29
+ const float beta2h = params[6];
30
+
31
+ const float gi = grad[i];
32
+ const float gmi = gradm[i]*beta1 + gi*(1.0f - beta1);
33
+ const float gvi = gradv[i]*beta2 + gi*gi*(1.0f - beta2);
34
+
35
+ gradm[i] = gmi;
36
+ gradv[i] = gvi;
37
+
38
+ const float mh = gmi*beta1h;
39
+ const float vh = sqrt(gvi*beta2h) + eps;
40
+
41
+ x[i] = x[i]*(1.0f - alpha*wd) - alpha*mh/vh;
42
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_sgd.comp ADDED
@@ -0,0 +1,22 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+
3
+ #include "generic_head.glsl"
4
+
5
+ layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
6
+
7
+ layout (binding = 0) buffer X {A_TYPE data_x[];};
8
+ layout (binding = 1) readonly buffer G {A_TYPE data_grad[];};
9
+ layout (binding = 2) readonly buffer P {float data_params[2];};
10
+
11
+ void main() {
12
+ const uint i = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;
13
+
14
+ if (i >= p.KX) {
15
+ return;
16
+ }
17
+
18
+ const float alpha = data_params[0];
19
+ const float keep = 1.f - alpha * data_params[1];
20
+
21
+ data_x[i] = data_x[i] * keep - alpha * data_grad[i];
22
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp ADDED
@@ -0,0 +1,64 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+
3
+ #include "types.glsl"
4
+
5
+ layout (push_constant) uniform parameter
6
+ {
7
+ uint ne;
8
+ uint ne00; uint ne01; uint ne02; uint ne03; uint nb00; uint nb01; uint nb02; uint nb03;
9
+ uint ne10; uint ne11; uint ne12; uint ne13; uint nb10; uint nb11; uint nb12; uint nb13;
10
+ uint misalign_offsets;
11
+ uint circular;
12
+
13
+ uint lp0; uint rp0;
14
+ uint lp1; uint rp1;
15
+ uint lp2; uint rp2;
16
+ uint lp3; uint rp3;
17
+ } p;
18
+
19
+ uint get_aoffset() { return p.misalign_offsets >> 16; }
20
+ uint get_doffset() { return p.misalign_offsets & 0xFFFF; }
21
+
22
+ uint wrap_around(int coord, uint size) {
23
+ return (uint(coord + int(size))) % size; // add size to avoid issues with negative
24
+ }
25
+
26
+ layout (binding = 0) readonly buffer A {A_TYPE data_a[];};
27
+ layout (binding = 1) writeonly buffer D {D_TYPE data_d[];};
28
+
29
+ layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
30
+
31
+ void main() {
32
+ const uint idx = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;
33
+
34
+ if (idx >= p.ne) {
35
+ return;
36
+ }
37
+
38
+ const uint i3 = idx / (p.ne12*p.ne11*p.ne10);
39
+ const uint i3_offset = i3 * p.ne12*p.ne11*p.ne10;
40
+ const uint i2 = (idx - i3_offset) / (p.ne11*p.ne10);
41
+ const uint i2_offset = i2*p.ne11*p.ne10;
42
+ const uint i1 = (idx - i3_offset - i2_offset) / p.ne10;
43
+ const uint i0 = idx - i3_offset - i2_offset - i1*p.ne10;
44
+
45
+ const uint src0_idx = (i3 - p.lp3)*p.nb03 + (i2 - p.lp2)*p.nb02 + (i1 - p.lp1)*p.nb01 + (i0 - p.lp0)*p.nb00;
46
+ const uint dst_idx = i3*p.nb13 + i2*p.nb12 + i1*p.nb11 + i0*p.nb10;
47
+
48
+ if (p.circular != 0u) {
49
+ const uint ci0 = wrap_around(int(i0) - int(p.lp0), p.ne00);
50
+ const uint ci1 = wrap_around(int(i1) - int(p.lp1), p.ne01);
51
+ const uint ci2 = wrap_around(int(i2) - int(p.lp2), p.ne02);
52
+ const uint ci3 = wrap_around(int(i3) - int(p.lp3), p.ne03);
53
+ const uint circular_src_idx = ci3*p.nb03 + ci2*p.nb02 + ci1*p.nb01 + ci0*p.nb00;
54
+ data_d[get_doffset() + dst_idx] = D_TYPE(data_a[get_aoffset() + circular_src_idx]);
55
+ } else {
56
+ const bool is_src0 = i0 >= p.lp0 && i0 < p.ne10 - p.rp0 &&
57
+ i1 >= p.lp1 && i1 < p.ne11 - p.rp1 &&
58
+ i2 >= p.lp2 && i2 < p.ne12 - p.rp2 &&
59
+ i3 >= p.lp3 && i3 < p.ne13 - p.rp3;
60
+ data_d[get_doffset() + dst_idx] = D_TYPE(is_src0 ? data_a[get_aoffset() + src0_idx] : 0.0f);
61
+ }
62
+
63
+
64
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp ADDED
@@ -0,0 +1,74 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+
3
+ #include "types.glsl"
4
+
5
+ #extension GL_EXT_shader_16bit_storage : require
6
+
7
+ layout(push_constant) uniform parameter {
8
+ uint IW; uint IH;
9
+ uint OW; uint OH;
10
+ uint OC;
11
+ uint pelements;
12
+ uint op;
13
+ int k0; int k1;
14
+ int s0; int s1;
15
+ int p0; int p1;
16
+ } p;
17
+
18
+ #define BLOCK_SIZE 512
19
+ #define FLT_MAX 3.402823466e+38F
20
+ #define OP_POOL_MAX 0u
21
+ #define OP_POOL_AVG 1u
22
+
23
+ layout (local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in;
24
+
25
+ layout(binding = 0) readonly buffer X {A_TYPE data_a[];};
26
+ layout(binding = 1) writeonly buffer D {D_TYPE data_d[];};
27
+
28
+ void main() {
29
+ const uint idx = gl_GlobalInvocationID.x;
30
+ if (idx >= p.pelements) {
31
+ return;
32
+ }
33
+
34
+ const uint O_HW = p.OW * p.OH;
35
+
36
+ const uint nc = idx / O_HW;
37
+ const uint cur_oh = (idx % O_HW) / p.OW;
38
+ const uint cur_ow = (idx % O_HW) % p.OW;
39
+
40
+ const int start_h = int(cur_oh) * p.s0 - p.p0;
41
+ const uint bh = max(start_h, 0);
42
+ const uint eh = min(start_h + p.k0, p.IH);
43
+
44
+ const int start_w = int(cur_ow) * p.s1 - p.p1;
45
+ const uint bw = max(start_w, 0);
46
+ const uint ew = min(start_w + p.k1, p.IW);
47
+
48
+ const float scale = 1.0 / float(p.k0 * p.k1);
49
+ float res;
50
+
51
+ if (p.op == OP_POOL_AVG) {
52
+ res = 0.0;
53
+ } else if (p.op == OP_POOL_MAX) {
54
+ res = -FLT_MAX;
55
+ } else {
56
+ return;
57
+ }
58
+
59
+ #pragma unroll
60
+ for (uint i = bh; i < eh; i++) {
61
+ #pragma unroll
62
+ for (uint j = bw; j < ew; j++) {
63
+ const float cur = D_TYPE(data_a[nc * p.IH * p.IW + i * p.IW + j]);
64
+
65
+ if (p.op == OP_POOL_AVG) {
66
+ res += cur * scale;
67
+ } else if (p.op == OP_POOL_MAX) {
68
+ res = max(res, cur);
69
+ }
70
+ }
71
+ }
72
+
73
+ data_d[nc * O_HW + cur_oh * p.OW + cur_ow] = res;
74
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp ADDED
@@ -0,0 +1,127 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+
3
+ #extension GL_EXT_control_flow_attributes : require
4
+ #extension GL_EXT_shader_16bit_storage : require
5
+
6
+ #ifdef USE_SUBGROUPS
7
+ #extension GL_KHR_shader_subgroup_basic : require
8
+ #extension GL_KHR_shader_subgroup_clustered : require
9
+
10
+ #define INVOCATION_ID gl_SubgroupInvocationID.x
11
+ #else
12
+ #define INVOCATION_ID gl_LocalInvocationID.x
13
+ #endif
14
+
15
+ layout (push_constant) uniform parameter
16
+ {
17
+ uint ne;
18
+ uint num_blocks;
19
+ } p;
20
+
21
+ #include "types.glsl"
22
+
23
+ layout(constant_id = 0) const uint GROUP_SIZE = 32;
24
+ layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
25
+
26
+ layout (binding = 0) readonly buffer A {vec4 data_a[];};
27
+ #ifndef QBLOCK_X4
28
+ layout (binding = 1) writeonly buffer D {block_q8_1_packed32 data_b[];};
29
+ #else
30
+ layout (binding = 1) writeonly buffer D {block_q8_1_x4 data_b[];};
31
+ #endif
32
+
33
+ #ifndef USE_SUBGROUPS
34
+ shared float shmem[GROUP_SIZE];
35
+ #endif
36
+
37
+ void quantize(const uint wgid) {
38
+ const uint tid = INVOCATION_ID;
39
+
40
+ // Each thread handles a vec4, so 8 threads handle a block
41
+ const uint blocks_per_group = GROUP_SIZE / 8;
42
+
43
+ const uint block_in_wg = tid / 8;
44
+
45
+ const uint ib = wgid * blocks_per_group + block_in_wg;
46
+ const uint iqs = tid % 8;
47
+
48
+ #ifdef QBLOCK_X4
49
+ const uint ibx4_outer = ib / 4;
50
+ const uint ibx4_inner = ib % 4;
51
+
52
+ const uint required_x4_blocks = (p.ne + 127) / 128;
53
+ if (ibx4_outer >= required_x4_blocks) {
54
+ return;
55
+ }
56
+ #endif
57
+
58
+ const uint a_idx = ib * 8 + iqs;
59
+
60
+ vec4 vals = a_idx < p.ne / 4 ? data_a[a_idx] : vec4(0.0f);
61
+ const vec4 abs_vals = abs(vals);
62
+
63
+ // Find absolute max for each block
64
+ const float thread_max = max(max(abs_vals.x, abs_vals.y), max(abs_vals.z, abs_vals.w));
65
+ #ifndef USE_SUBGROUPS
66
+ shmem[tid] = thread_max;
67
+ barrier();
68
+ [[unroll]] for (uint s = 4; s > 0; s >>= 1) {
69
+ if (iqs < s) {
70
+ shmem[tid] = max(shmem[tid], shmem[tid + s]);
71
+ }
72
+ barrier();
73
+ }
74
+
75
+ const float amax = shmem[block_in_wg * 8];
76
+ #else
77
+ const float amax = subgroupClusteredMax(thread_max, 8);
78
+ #endif
79
+
80
+ const float d = amax / 127.0;
81
+ const float d_inv = d != 0.0 ? 1.0 / d : 0.0;
82
+ vals = round(vals * d_inv);
83
+
84
+ #ifndef QBLOCK_X4
85
+ data_b[ib].qs[iqs] = pack32(i8vec4(round(vals)));
86
+ #else
87
+ data_b[ibx4_outer].qs[ibx4_inner * 8 + iqs] = pack32(i8vec4(round(vals)));
88
+ #endif
89
+
90
+ #ifndef USE_SUBGROUPS
91
+ barrier();
92
+ #endif
93
+
94
+ // Calculate the sum for each block
95
+ const float thread_sum = vals.x + vals.y + vals.z + vals.w;
96
+ #ifndef USE_SUBGROUPS
97
+ shmem[tid] = thread_sum;
98
+ barrier();
99
+ [[unroll]] for (uint s = 4; s > 0; s >>= 1) {
100
+ if (iqs < s) {
101
+ shmem[tid] += shmem[tid + s];
102
+ }
103
+ barrier();
104
+ }
105
+ #else
106
+ const float sum = subgroupClusteredAdd(thread_sum, 8);
107
+ #endif
108
+ if (iqs == 0) {
109
+ #ifndef USE_SUBGROUPS
110
+ const float sum = shmem[tid];
111
+ #endif
112
+
113
+ #ifndef QBLOCK_X4
114
+ data_b[ib].ds = f16vec2(vec2(d, sum * d));
115
+ #else
116
+ data_b[ibx4_outer].ds[ibx4_inner] = f16vec2(vec2(d, sum * d));
117
+ #endif
118
+ }
119
+ }
120
+
121
+ void main() {
122
+ uint wgid = gl_WorkGroupID.x;
123
+ while (wgid < p.num_blocks) {
124
+ quantize(wgid);
125
+ wgid += gl_NumWorkGroups.x;
126
+ }
127
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/reglu.comp ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+
3
+ #include "glu_head.glsl"
4
+
5
+ float op(float a, float b) {
6
+ return max(a, 0.0f) * b;
7
+ }
8
+
9
+ #include "glu_main.glsl"
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp ADDED
@@ -0,0 +1,26 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+
3
+ #include "types.glsl"
4
+ #include "generic_unary_head.glsl"
5
+
6
+ layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
7
+
8
+ uint src0_idx_mod(uint idx) {
9
+ const uint i13 = idx / (p.ne12*p.ne11*p.ne10);
10
+ const uint i13_offset = i13 * p.ne12*p.ne11*p.ne10;
11
+ const uint i12 = (idx - i13_offset) / (p.ne11*p.ne10);
12
+ const uint i12_offset = i12*p.ne11*p.ne10;
13
+ const uint i11 = (idx - i13_offset - i12_offset) / p.ne10;
14
+ const uint i10 = idx - i13_offset - i12_offset - i11*p.ne10;
15
+ return (i13 % p.ne03)*p.nb03 + (i12 % p.ne02)*p.nb02 + (i11 % p.ne01)*p.nb01 + (i10 % p.ne00)*p.nb00;
16
+ }
17
+
18
+ void main() {
19
+ const uint idx = get_idx();
20
+
21
+ if (idx >= p.ne) {
22
+ return;
23
+ }
24
+
25
+ data_d[get_doffset() + dst_idx(idx)] = D_TYPE(data_a[get_aoffset() + src0_idx_mod(idx)]);
26
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp ADDED
@@ -0,0 +1,37 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+
3
+ #include "types.glsl"
4
+ #include "generic_unary_head.glsl"
5
+
6
+ layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
7
+
8
+ void main() {
9
+ const uint idx = get_idx();
10
+
11
+ if (idx >= p.ne) {
12
+ return;
13
+ }
14
+
15
+ // Destination multi-index (inlined dst_idx)
16
+ const uint i13 = fastdiv(idx, p.ne1_012mp, fastdiv_L(p.ne1_Ls, 0));
17
+ const uint i13_offset = i13 * p.ne12*p.ne11*p.ne10;
18
+ const uint i12 = fastdiv(idx - i13_offset, p.ne1_01mp, fastdiv_L(p.ne1_Ls, 1));
19
+ const uint i12_offset = i12*p.ne11*p.ne10;
20
+ const uint i11 = fastdiv(idx - i13_offset - i12_offset, p.ne1_0mp, fastdiv_L(p.ne1_Ls, 2));
21
+ const uint i10 = idx - i13_offset - i12_offset - i11*p.ne10;
22
+ const uint d_idx = i13*p.nb13 + i12*p.nb12 + i11*p.nb11 + i10*p.nb10;
23
+
24
+ // Accumulate from sources
25
+ A_TYPE acc = A_TYPE(0);
26
+ for (uint i3 = i13; i3 < p.ne03; i3 += p.ne13) {
27
+ for (uint i2 = i12; i2 < p.ne02; i2 += p.ne12) {
28
+ for (uint i1 = i11; i1 < p.ne01; i1 += p.ne11) {
29
+ for (uint i0 = i10; i0 < p.ne00; i0 += p.ne10) {
30
+ acc += data_a[i3*p.nb03 + i2*p.nb02 + i1*p.nb01 + i0*p.nb00];
31
+ }
32
+ }
33
+ }
34
+ }
35
+
36
+ data_d[get_doffset() + d_idx] = D_TYPE(acc);
37
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp ADDED
@@ -0,0 +1,150 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+
3
+ #include "generic_binary_head.glsl"
4
+ #include "types.glsl"
5
+
6
+ #if RMS_NORM_ROPE_FUSION
7
+
8
+ layout (binding = 0) readonly buffer A {A_TYPE data_a[];};
9
+ layout (binding = 1) readonly buffer B {B_TYPE data_b[];};
10
+
11
+ // data is passed from rms_norm -> rope through shared memory.
12
+ // rms_norm calls this data_d, rope calls this rope_data_a.
13
+ // Binding 2 is not used
14
+ shared FLOAT_TYPE rope_data_a[1024];
15
+ #define data_d rope_data_a
16
+
17
+ layout (binding = 3) readonly buffer R_Y {int rope_data_pos[];};
18
+ layout (binding = 4) readonly buffer R_Z {float rope_data_ff[];};
19
+ layout (binding = 5) writeonly buffer R_D {ROPE_D_TYPE rope_data_d[];};
20
+ layout (binding = 6) readonly buffer R_I {uvec2 rope_data_i[];}; // indices for set_rows
21
+
22
+ #include "rope_params.glsl"
23
+ #include "rope_funcs.glsl"
24
+
25
+ #define GGML_ROPE_TYPE_NORMAL 0
26
+ #define GGML_ROPE_TYPE_NEOX 2
27
+ #define GGML_ROPE_TYPE_MROPE 8
28
+ #define GGML_ROPE_TYPE_VISION 24
29
+
30
+ #endif
31
+
32
+ #extension GL_EXT_control_flow_attributes : enable
33
+ #define BLOCK_SIZE 512
34
+
35
+ layout (constant_id = 1) const bool do_multiply = false;
36
+
37
+ layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in;
38
+
39
+ shared FLOAT_TYPE sumsh[BLOCK_SIZE];
40
+
41
+ void rms_norm(uint num_iters) {
42
+ const uint ncols = p.ne00;
43
+ const uint nrows = gl_NumWorkGroups.x;
44
+ const uint nchannels = gl_NumWorkGroups.y;
45
+
46
+ const uint row = gl_WorkGroupID.x;
47
+ const uint channel = gl_WorkGroupID.y;
48
+ const uint samp = gl_WorkGroupID.z;
49
+ const uint tid = gl_LocalInvocationID.x;
50
+
51
+ const uint stride_row = p.nb01;
52
+ const uint stride_channel = p.nb02;
53
+ const uint stride_sample = p.nb03;
54
+
55
+ uint32_t a_offset = samp*stride_sample + channel*stride_channel + row*stride_row + get_aoffset();
56
+ uint32_t b_offset = src1_idx(0, row, channel, samp) + get_boffset();
57
+ #if RMS_NORM_ROPE_FUSION
58
+ // Per-row offset in shared memory
59
+ uint32_t d_offset = 0;
60
+ #else
61
+ uint32_t d_offset = ((samp*nchannels + channel)*nrows + row)*ncols + get_doffset();
62
+ #endif
63
+ FLOAT_TYPE sum = FLOAT_TYPE(0.0f); // partial sum for thread in warp
64
+
65
+ [[unroll]] for (uint col = tid, idx = 0; idx < num_iters; col += BLOCK_SIZE, ++idx) {
66
+ FLOAT_TYPE xi = FLOAT_TYPE(0);
67
+ if (col < ncols) {
68
+ xi = FLOAT_TYPE(data_a[a_offset + col]);
69
+ }
70
+ sum += xi * xi;
71
+ }
72
+
73
+ sumsh[tid] = sum;
74
+ // sum up partial sums and write back result
75
+ barrier();
76
+ [[unroll]] for (int s = BLOCK_SIZE / 2; s > 0; s >>= 1) {
77
+ if (tid < s) {
78
+ sum += sumsh[tid + s];
79
+ sumsh[tid] = sum;
80
+ }
81
+ barrier();
82
+ }
83
+ sum = sumsh[0];
84
+
85
+ const FLOAT_TYPE mean = sum / FLOAT_TYPE(ncols);
86
+ const FLOAT_TYPE scale = inversesqrt(mean + FLOAT_TYPE(p.param1));
87
+
88
+ if (do_multiply) {
89
+ if (ncols > p.ne10) {
90
+ [[unroll]] for (uint col = tid, idx = 0; idx < num_iters; col += BLOCK_SIZE, ++idx) {
91
+ if (col >= ncols) {
92
+ continue;
93
+ }
94
+ data_d[d_offset + col] = D_TYPE(scale * FLOAT_TYPE(data_a[a_offset + col]) * FLOAT_TYPE(data_b[b_offset + fastmod(col, p.ne10)]));
95
+ }
96
+ } else {
97
+ [[unroll]] for (uint col = tid, idx = 0; idx < num_iters; col += BLOCK_SIZE, ++idx) {
98
+ if (col >= ncols) {
99
+ continue;
100
+ }
101
+ data_d[d_offset + col] = D_TYPE(scale * FLOAT_TYPE(data_a[a_offset + col]) * FLOAT_TYPE(data_b[b_offset + col]));
102
+ }
103
+ }
104
+ } else {
105
+ [[unroll]] for (uint col = tid, idx = 0; idx < num_iters; col += BLOCK_SIZE, ++idx) {
106
+ if (col >= ncols) {
107
+ continue;
108
+ }
109
+ data_d[d_offset + col] = D_TYPE(scale * FLOAT_TYPE(data_a[a_offset + col]));
110
+ }
111
+ }
112
+ #if RMS_NORM_ROPE_FUSION
113
+ barrier();
114
+ rope_params rp = p.rope;
115
+ for (uint t = 2*tid; t < ncols; t += 2*BLOCK_SIZE) {
116
+ if (rp.rope_mode == GGML_ROPE_TYPE_NEOX) {
117
+ rope_neox(t, row, channel, samp, rp);
118
+ } else if (rp.rope_mode == GGML_ROPE_TYPE_NORMAL) {
119
+ rope_norm(t, row, channel, samp, rp);
120
+ }
121
+ }
122
+ #endif
123
+ }
124
+
125
+ void main() {
126
+ // instantiate the rms_norm function for several different
127
+ // dimensions, to allow loop unrolling
128
+ uint num_blocks = (p.ne00 + BLOCK_SIZE - 1) / BLOCK_SIZE;
129
+ if (num_blocks > 32) {
130
+ rms_norm(num_blocks);
131
+ } else if (num_blocks > 16) {
132
+ rms_norm(32);
133
+ } else if (num_blocks > 12) {
134
+ rms_norm(16);
135
+ } else if (num_blocks > 10) {
136
+ rms_norm(12);
137
+ } else if (num_blocks > 8) {
138
+ rms_norm(10);
139
+ } else if (num_blocks > 4) {
140
+ rms_norm(8);
141
+ } else if (num_blocks == 4) {
142
+ rms_norm(4);
143
+ } else if (num_blocks == 3) {
144
+ rms_norm(3);
145
+ } else if (num_blocks == 2) {
146
+ rms_norm(2);
147
+ } else if (num_blocks == 1) {
148
+ rms_norm(1);
149
+ }
150
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp ADDED
@@ -0,0 +1,55 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+
3
+ #include "generic_head.glsl"
4
+ #include "types.glsl"
5
+
6
+ #extension GL_EXT_control_flow_attributes : enable
7
+ #define BLOCK_SIZE 512
8
+
9
+ layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in;
10
+
11
+ layout (binding = 0) readonly buffer G {A_TYPE data_a[];};
12
+ layout (binding = 1) readonly buffer X {B_TYPE data_b[];};
13
+ layout (binding = 2) writeonly buffer D {D_TYPE data_d[];};
14
+
15
+ shared FLOAT_TYPE sum_xx[BLOCK_SIZE];
16
+ shared FLOAT_TYPE sum_xg[BLOCK_SIZE];
17
+
18
+ void main() {
19
+ const uint row = gl_WorkGroupID.z * 262144 + gl_WorkGroupID.y * 512 + gl_WorkGroupID.x;
20
+ const uint tid = gl_LocalInvocationID.x;
21
+
22
+ // Compute derivative of x[i]/norm(x) = g[i]/norm(x) - x[i] dot(x,g)/KX / norm(x)^1.5
23
+
24
+ // partial sums for thread in warp
25
+ sum_xx[tid] = FLOAT_TYPE(0.0f);
26
+ sum_xg[tid] = FLOAT_TYPE(0.0f);
27
+
28
+ [[unroll]] for (uint col = tid; col < p.KX; col += BLOCK_SIZE) {
29
+ const FLOAT_TYPE gi = FLOAT_TYPE(data_a[row*p.KX + col]);
30
+ const FLOAT_TYPE xi = FLOAT_TYPE(data_b[row*p.KX + col]);
31
+ sum_xx[tid] += xi * xi;
32
+ sum_xg[tid] += xi * gi;
33
+ }
34
+
35
+ // sum up partial sums and write back result
36
+ barrier();
37
+ [[unroll]] for (int s = BLOCK_SIZE / 2; s > 0; s >>= 1) {
38
+ if (tid < s) {
39
+ sum_xx[tid] += sum_xx[tid + s];
40
+ sum_xg[tid] += sum_xg[tid + s];
41
+ }
42
+ barrier();
43
+ }
44
+
45
+ const FLOAT_TYPE eps = FLOAT_TYPE(p.param1);
46
+ const FLOAT_TYPE mean = sum_xx[0] / FLOAT_TYPE(p.KX);
47
+ const FLOAT_TYPE scale_g = inversesqrt(mean + eps);
48
+ const FLOAT_TYPE scale_x = -scale_g * sum_xg[0] / (sum_xx[0] + FLOAT_TYPE(p.KX) * eps);
49
+
50
+ [[unroll]] for (uint col = tid; col < p.KX; col += BLOCK_SIZE) {
51
+ data_d[row*p.KX + col] = D_TYPE(
52
+ scale_g * FLOAT_TYPE(data_a[row*p.KX + col]) +
53
+ scale_x * FLOAT_TYPE(data_b[row*p.KX + col]));
54
+ }
55
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+
3
+ #include "generic_binary_head.glsl"
4
+ #include "types.glsl"
5
+
6
+ #extension GL_EXT_control_flow_attributes : enable
7
+ #extension GL_KHR_shader_subgroup_arithmetic : enable
8
+ #extension GL_KHR_shader_subgroup_basic : enable
9
+
10
+ #define BLOCK_SIZE 128
11
+
12
+ layout (constant_id = 1) const bool do_multiply = false;
13
+
14
+ layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in;
15
+
16
+ layout (binding = 3, std430) readonly buffer PartialsBuf {float partial_sums[];};
17
+
18
+ shared FLOAT_TYPE sumsh[BLOCK_SIZE];
19
+
20
+ void main() {
21
+ const uint ncols = p.ne00;
22
+ const uint nrows = gl_NumWorkGroups.x;
23
+ const uint nchannels = gl_NumWorkGroups.y;
24
+
25
+ const uint row = 0;
26
+ const uint channel = gl_WorkGroupID.y;
27
+ const uint samp = gl_WorkGroupID.z;
28
+ // The work is split across multiple workgroups in the x dimension. Each invocation
29
+ // processes one element
30
+ const uint tid = gl_GlobalInvocationID.x;
31
+
32
+ const uint stride_row = p.nb01;
33
+ const uint stride_channel = p.nb02;
34
+ const uint stride_sample = p.nb03;
35
+
36
+ uint32_t a_offset = samp*stride_sample + channel*stride_channel + row*stride_row + get_aoffset();
37
+ uint32_t b_offset = src1_idx(0, row, channel, samp) + get_boffset();
38
+ uint32_t d_offset = ((samp*nchannels + channel)*nrows + row)*ncols + get_doffset();
39
+
40
+ FLOAT_TYPE sum = FLOAT_TYPE(0.0f); // partial sum for thread in warp
41
+
42
+ uint32_t num_partials = p.param3;
43
+ for (uint32_t i = gl_SubgroupInvocationID; i < num_partials; i += gl_SubgroupSize) {
44
+ sum += partial_sums[i];
45
+ }
46
+ sum = subgroupAdd(sum);
47
+
48
+ uint col = tid;
49
+ if (col >= ncols) {
50
+ return;
51
+ }
52
+
53
+ const FLOAT_TYPE mean = sum / FLOAT_TYPE(ncols);
54
+ const FLOAT_TYPE scale = inversesqrt(mean + FLOAT_TYPE(p.param1));
55
+
56
+ if (do_multiply) {
57
+ if (ncols > p.ne10) {
58
+ data_d[d_offset + col] = D_TYPE(scale * FLOAT_TYPE(data_a[a_offset + col]) * FLOAT_TYPE(data_b[b_offset + fastmod(col, p.ne10)]));
59
+ } else {
60
+ data_d[d_offset + col] = D_TYPE(scale * FLOAT_TYPE(data_a[a_offset + col]) * FLOAT_TYPE(data_b[b_offset + col]));
61
+ }
62
+ } else {
63
+ data_d[d_offset + col] = D_TYPE(scale * FLOAT_TYPE(data_a[a_offset + col]));
64
+ }
65
+ }
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/roll.comp ADDED
@@ -0,0 +1,46 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #version 450
2
+
3
+ #include "types.glsl"
4
+ #include "generic_unary_head.glsl"
5
+
6
+ layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
7
+
8
+ uint wrap_idx(int i, uint ne) {
9
+ if (i < 0) {
10
+ return i + ne;
11
+ } else if (i >= ne) {
12
+ return i - ne;
13
+ }
14
+ return i;
15
+ }
16
+
17
+ void main() {
18
+ const uint idx = get_idx();
19
+ if (idx >= p.ne) {
20
+ return;
21
+ }
22
+
23
+ const uint i3 = fastdiv(idx, p.ne1_012mp, fastdiv_L(p.ne1_Ls, 0));
24
+ const uint i3_offset = i3 * p.ne12*p.ne11*p.ne10;
25
+ const uint i2 = fastdiv(idx - i3_offset, p.ne1_01mp, fastdiv_L(p.ne1_Ls, 1));
26
+ const uint i2_offset = i2*p.ne11*p.ne10;
27
+ const uint i1 = fastdiv(idx - i3_offset - i2_offset, p.ne1_0mp, fastdiv_L(p.ne1_Ls, 2));
28
+ const uint i0 = idx - i3_offset - i2_offset - i1*p.ne10;
29
+
30
+ const uint p1 = floatBitsToUint(p.param1);
31
+ const uint p2 = floatBitsToUint(p.param2);
32
+ const int s0 = int(p1 >> 16) - 0x8000;
33
+ const int s1 = int(p1 & 0xFFFF) - 0x8000;
34
+ const int s2 = int(p2 >> 16) - 0x8000;
35
+ const int s3 = int(p2 & 0xFFFF) - 0x8000;
36
+
37
+ const uint i00 = wrap_idx(int(i0) - s0, p.ne10);
38
+ const uint i01 = wrap_idx(int(i1) - s1, p.ne11);
39
+ const uint i02 = wrap_idx(int(i2) - s2, p.ne12);
40
+ const uint i03 = wrap_idx(int(i3) - s3, p.ne13);
41
+
42
+ const uint a_idx = i03*p.nb03 + i02*p.nb02 + i01*p.nb01 + i00*p.nb00;
43
+ const uint d_idx = i3 *p.nb13 + i2 *p.nb12 + i1 *p.nb11 + i0 *p.nb10;
44
+
45
+ data_d[get_doffset() + d_idx] = D_TYPE(data_a[get_aoffset() + a_idx]);
46
+ }