Safetensors
GGUF
Turkish
llama
Llama-3
instruct
finetune
chatml
gpt4
synthetic data
distillation
function calling
json mode
axolotl
roleplaying
chat
Instructions to use tda45/TdAI with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- llama.cpp
How to use tda45/TdAI with llama.cpp:
Install (macOS, Linux)
curl -LsSf https://llama.app/install.sh | sh # Start a local OpenAI-compatible server with a web UI: llama serve -hf tda45/TdAI # Run inference directly in the terminal: llama cli -hf tda45/TdAI
Install from WinGet (Windows)
winget install llama.cpp # Start a local OpenAI-compatible server with a web UI: llama serve -hf tda45/TdAI # Run inference directly in the terminal: llama cli -hf tda45/TdAI
Build from source code
git clone https://github.com/ggml-org/llama.cpp.git cd llama.cpp cmake -B build cmake --build build -j --target llama-app # Start a local OpenAI-compatible server with a web UI: ./build/bin/llama serve -hf tda45/TdAI # Run inference directly in the terminal: ./build/bin/llama cli -hf tda45/TdAI
Use Docker
docker model run hf.co/tda45/TdAI
- Llama new
- LM Studio
- Jan
- Ollama
How to use tda45/TdAI with Ollama:
ollama run hf.co/tda45/TdAI
- Unsloth Desktop
- Docker Model Runner
How to use tda45/TdAI with Docker Model Runner:
docker model run hf.co/tda45/TdAI
- Lemonade
How to use tda45/TdAI with Lemonade:
Pull the model
# Download Lemonade from https://lemonade-server.ai/ lemonade pull tda45/TdAI
Run and chat with the model
lemonade run user.TdAI-{{QUANT_TAG}}List all available models
lemonade list
- Atomic Chat
Upload folder using huggingface_hub (part 6)
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .gitattributes +3 -0
- llama.cpp/LICENSE +21 -0
- llama.cpp/Makefile +9 -0
- llama.cpp/README.md +604 -0
- llama.cpp/SECURITY.md +97 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp +124 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp +41 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/log.comp +17 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp +27 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp +48 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp +264 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl +230 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl +35 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp +132 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp +95 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp +90 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp +105 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp +87 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp +90 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp +88 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp +124 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp +156 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp +128 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp +132 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp +134 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp +165 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp +130 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp +141 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl +527 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp +466 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp +658 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl +617 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl +74 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +311 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl +454 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl +93 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/multi_add.comp +194 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp +44 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp +42 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_sgd.comp +22 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp +64 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp +74 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp +127 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/reglu.comp +9 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp +26 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp +37 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp +150 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp +55 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp +65 -0
- llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/roll.comp +46 -0
.gitattributes
CHANGED
|
@@ -36,3 +36,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
|
|
| 36 |
*.gguf filter=lfs diff=lfs merge=lfs -text
|
| 37 |
llama.cpp/docs/android/imported-into-android-studio.jpg filter=lfs diff=lfs merge=lfs -text
|
| 38 |
llama.cpp/docs/development/llama-star/idea-arch.key filter=lfs diff=lfs merge=lfs -text
|
|
|
|
|
|
|
|
|
|
|
|
| 36 |
*.gguf filter=lfs diff=lfs merge=lfs -text
|
| 37 |
llama.cpp/docs/android/imported-into-android-studio.jpg filter=lfs diff=lfs merge=lfs -text
|
| 38 |
llama.cpp/docs/development/llama-star/idea-arch.key filter=lfs diff=lfs merge=lfs -text
|
| 39 |
+
llama.cpp/media/llama0-banner.png filter=lfs diff=lfs merge=lfs -text
|
| 40 |
+
llama.cpp/media/llama0-logo.png filter=lfs diff=lfs merge=lfs -text
|
| 41 |
+
llama.cpp/media/matmul.png filter=lfs diff=lfs merge=lfs -text
|
llama.cpp/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
MIT License
|
| 2 |
+
|
| 3 |
+
Copyright (c) 2023-2026 The ggml authors
|
| 4 |
+
|
| 5 |
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
| 6 |
+
of this software and associated documentation files (the "Software"), to deal
|
| 7 |
+
in the Software without restriction, including without limitation the rights
|
| 8 |
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
| 9 |
+
copies of the Software, and to permit persons to whom the Software is
|
| 10 |
+
furnished to do so, subject to the following conditions:
|
| 11 |
+
|
| 12 |
+
The above copyright notice and this permission notice shall be included in all
|
| 13 |
+
copies or substantial portions of the Software.
|
| 14 |
+
|
| 15 |
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
| 16 |
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
| 17 |
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
| 18 |
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
| 19 |
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
| 20 |
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
| 21 |
+
SOFTWARE.
|
llama.cpp/Makefile
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
define newline
|
| 2 |
+
|
| 3 |
+
|
| 4 |
+
endef
|
| 5 |
+
|
| 6 |
+
$(error Build system changed:$(newline)\
|
| 7 |
+
The Makefile build has been replaced by CMake.$(newline)$(newline)\
|
| 8 |
+
For build instructions see:$(newline)\
|
| 9 |
+
https://github.com/ggml-org/llama.cpp/blob/master/docs/build.md$(newline)${newline})
|
llama.cpp/README.md
ADDED
|
@@ -0,0 +1,604 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# llama.cpp
|
| 2 |
+
|
| 3 |
+

|
| 4 |
+
|
| 5 |
+
[](https://opensource.org/licenses/MIT)
|
| 6 |
+
[](https://github.com/ggml-org/llama.cpp/releases)
|
| 7 |
+
[](https://github.com/ggml-org/llama.cpp/actions/workflows/server.yml)
|
| 8 |
+
[](https://github.com/ggml-org/llama.cpp/actions/workflows/docker.yml)
|
| 9 |
+
[](https://github.com/ggml-org/llama.cpp/actions/workflows/winget.yml)
|
| 10 |
+
|
| 11 |
+
[Manifesto](https://github.com/ggml-org/llama.cpp/discussions/205) / [ggml](https://github.com/ggml-org/ggml) / [ops](https://github.com/ggml-org/llama.cpp/blob/master/docs/ops.md)
|
| 12 |
+
|
| 13 |
+
LLM inference in C/C++
|
| 14 |
+
|
| 15 |
+
## Recent API changes
|
| 16 |
+
|
| 17 |
+
- [Changelog for `libllama` API](https://github.com/ggml-org/llama.cpp/issues/9289)
|
| 18 |
+
- [Changelog for `llama-server` REST API](https://github.com/ggml-org/llama.cpp/issues/9291)
|
| 19 |
+
|
| 20 |
+
## Hot topics
|
| 21 |
+
|
| 22 |
+
- **Hugging Face cache migration: models downloaded with `-hf` are now stored in the standard Hugging Face cache directory, enabling sharing with other HF tools.**
|
| 23 |
+
- **[guide : using the new WebUI of llama.cpp](https://github.com/ggml-org/llama.cpp/discussions/16938)**
|
| 24 |
+
- [guide : running gpt-oss with llama.cpp](https://github.com/ggml-org/llama.cpp/discussions/15396)
|
| 25 |
+
- [[FEEDBACK] Better packaging for llama.cpp to support downstream consumers 🤗](https://github.com/ggml-org/llama.cpp/discussions/15313)
|
| 26 |
+
- Support for the `gpt-oss` model with native MXFP4 format has been added | [PR](https://github.com/ggml-org/llama.cpp/pull/15091) | [Collaboration with NVIDIA](https://blogs.nvidia.com/blog/rtx-ai-garage-openai-oss) | [Comment](https://github.com/ggml-org/llama.cpp/discussions/15095)
|
| 27 |
+
- Multimodal support arrived in `llama-server`: [#12898](https://github.com/ggml-org/llama.cpp/pull/12898) | [documentation](./docs/multimodal.md)
|
| 28 |
+
- VS Code extension for FIM completions: https://github.com/ggml-org/llama.vscode
|
| 29 |
+
- Vim/Neovim plugin for FIM completions: https://github.com/ggml-org/llama.vim
|
| 30 |
+
- Hugging Face Inference Endpoints now support GGUF out of the box! https://github.com/ggml-org/llama.cpp/discussions/9669
|
| 31 |
+
- Hugging Face GGUF editor: [discussion](https://github.com/ggml-org/llama.cpp/discussions/9268) | [tool](https://huggingface.co/spaces/CISCai/gguf-editor)
|
| 32 |
+
- WebGPU support is now available in the browser, see a blog/demo introducing it [here](https://reeselevine.github.io/llamas-on-the-web/).
|
| 33 |
+
|
| 34 |
+
----
|
| 35 |
+
|
| 36 |
+
## Quick start
|
| 37 |
+
|
| 38 |
+
Getting started with llama.cpp is straightforward. Here are several ways to install it on your machine:
|
| 39 |
+
|
| 40 |
+
- Install `llama.cpp` using [brew, nix, winget, or conda-forge](docs/install.md)
|
| 41 |
+
- Run with Docker - see our [Docker documentation](docs/docker.md)
|
| 42 |
+
- Download pre-built binaries from the [releases page](https://github.com/ggml-org/llama.cpp/releases)
|
| 43 |
+
- Build from source by cloning this repository - check out [our build guide](docs/build.md)
|
| 44 |
+
|
| 45 |
+
Once installed, you'll need a model to work with. Head to the [Obtaining and quantizing models](#obtaining-and-quantizing-models) section to learn more.
|
| 46 |
+
|
| 47 |
+
Example command:
|
| 48 |
+
|
| 49 |
+
```sh
|
| 50 |
+
# Use a local model file
|
| 51 |
+
llama-cli -m my_model.gguf
|
| 52 |
+
|
| 53 |
+
# Or download and run a model directly from Hugging Face
|
| 54 |
+
llama-cli -hf ggml-org/gemma-3-1b-it-GGUF
|
| 55 |
+
|
| 56 |
+
# Launch OpenAI-compatible API server
|
| 57 |
+
llama-server -hf ggml-org/gemma-3-1b-it-GGUF
|
| 58 |
+
```
|
| 59 |
+
|
| 60 |
+
## Description
|
| 61 |
+
|
| 62 |
+
The main goal of `llama.cpp` is to enable LLM inference with minimal setup and state-of-the-art performance on a wide
|
| 63 |
+
range of hardware - locally and in the cloud.
|
| 64 |
+
|
| 65 |
+
- Plain C/C++ implementation without any dependencies
|
| 66 |
+
- Apple silicon is a first-class citizen - optimized via ARM NEON, Accelerate and Metal frameworks
|
| 67 |
+
- AVX, AVX2, AVX512 and AMX support for x86 architectures
|
| 68 |
+
- RVV, ZVFH, ZFH, ZICBOP and ZIHINTPAUSE support for RISC-V architectures
|
| 69 |
+
- 1.5-bit, 2-bit, 3-bit, 4-bit, 5-bit, 6-bit, and 8-bit integer quantization for faster inference and reduced memory use
|
| 70 |
+
- Custom CUDA kernels for running LLMs on NVIDIA GPUs (support for AMD GPUs via HIP and Moore Threads GPUs via MUSA)
|
| 71 |
+
- Vulkan and SYCL backend support
|
| 72 |
+
- CPU+GPU hybrid inference to partially accelerate models larger than the total VRAM capacity
|
| 73 |
+
|
| 74 |
+
The `llama.cpp` project is the main playground for developing new features for the [ggml](https://github.com/ggml-org/ggml) library.
|
| 75 |
+
|
| 76 |
+
<details>
|
| 77 |
+
<summary>Models</summary>
|
| 78 |
+
|
| 79 |
+
Typically finetunes of the base models below are supported as well.
|
| 80 |
+
|
| 81 |
+
Instructions for adding support for new models: [HOWTO-add-model.md](docs/development/HOWTO-add-model.md)
|
| 82 |
+
|
| 83 |
+
#### Text-only
|
| 84 |
+
|
| 85 |
+
- [X] LLaMA 🦙
|
| 86 |
+
- [x] LLaMA 2 🦙🦙
|
| 87 |
+
- [x] LLaMA 3 🦙🦙🦙
|
| 88 |
+
- [X] [Mistral 7B](https://huggingface.co/mistralai/Mistral-7B-v0.1)
|
| 89 |
+
- [x] [Mixtral MoE](https://huggingface.co/models?search=mistral-ai/Mixtral)
|
| 90 |
+
- [x] [DBRX](https://huggingface.co/databricks/dbrx-instruct)
|
| 91 |
+
- [x] [Jamba](https://huggingface.co/ai21labs)
|
| 92 |
+
- [X] [Falcon](https://huggingface.co/models?search=tiiuae/falcon)
|
| 93 |
+
- [X] [Chinese LLaMA / Alpaca](https://github.com/ymcui/Chinese-LLaMA-Alpaca) and [Chinese LLaMA-2 / Alpaca-2](https://github.com/ymcui/Chinese-LLaMA-Alpaca-2)
|
| 94 |
+
- [X] [Vigogne (French)](https://github.com/bofenghuang/vigogne)
|
| 95 |
+
- [X] [BERT](https://github.com/ggml-org/llama.cpp/pull/5423)
|
| 96 |
+
- [X] [Koala](https://bair.berkeley.edu/blog/2023/04/03/koala/)
|
| 97 |
+
- [X] [Baichuan 1 & 2](https://huggingface.co/models?search=baichuan-inc/Baichuan) + [derivations](https://huggingface.co/hiyouga/baichuan-7b-sft)
|
| 98 |
+
- [X] [Aquila 1 & 2](https://huggingface.co/models?search=BAAI/Aquila)
|
| 99 |
+
- [X] [Starcoder models](https://github.com/ggml-org/llama.cpp/pull/3187)
|
| 100 |
+
- [X] [Refact](https://huggingface.co/smallcloudai/Refact-1_6B-fim)
|
| 101 |
+
- [X] [MPT](https://github.com/ggml-org/llama.cpp/pull/3417)
|
| 102 |
+
- [X] [Bloom](https://github.com/ggml-org/llama.cpp/pull/3553)
|
| 103 |
+
- [x] [Yi models](https://huggingface.co/models?search=01-ai/Yi)
|
| 104 |
+
- [X] [StableLM models](https://huggingface.co/stabilityai)
|
| 105 |
+
- [x] [Deepseek models](https://huggingface.co/models?search=deepseek-ai/deepseek)
|
| 106 |
+
- [x] [Qwen models](https://huggingface.co/models?search=Qwen/Qwen)
|
| 107 |
+
- [x] [PLaMo-13B](https://github.com/ggml-org/llama.cpp/pull/3557)
|
| 108 |
+
- [x] [Phi models](https://huggingface.co/models?search=microsoft/phi)
|
| 109 |
+
- [x] [PhiMoE](https://github.com/ggml-org/llama.cpp/pull/11003)
|
| 110 |
+
- [x] [GPT-2](https://huggingface.co/gpt2)
|
| 111 |
+
- [x] [Orion 14B](https://github.com/ggml-org/llama.cpp/pull/5118)
|
| 112 |
+
- [x] [InternLM2](https://huggingface.co/models?search=internlm2)
|
| 113 |
+
- [x] [CodeShell](https://github.com/WisdomShell/codeshell)
|
| 114 |
+
- [x] [Gemma](https://ai.google.dev/gemma)
|
| 115 |
+
- [x] [Mamba](https://github.com/state-spaces/mamba)
|
| 116 |
+
- [x] [Grok-1](https://huggingface.co/keyfan/grok-1-hf)
|
| 117 |
+
- [x] [Xverse](https://huggingface.co/models?search=xverse)
|
| 118 |
+
- [x] [Command-R models](https://huggingface.co/models?search=CohereForAI/c4ai-command-r)
|
| 119 |
+
- [x] [SEA-LION](https://huggingface.co/models?search=sea-lion)
|
| 120 |
+
- [x] [GritLM-7B](https://huggingface.co/GritLM/GritLM-7B) + [GritLM-8x7B](https://huggingface.co/GritLM/GritLM-8x7B)
|
| 121 |
+
- [x] [OLMo](https://allenai.org/olmo)
|
| 122 |
+
- [x] [OLMo 2](https://allenai.org/olmo)
|
| 123 |
+
- [x] [OLMoE](https://huggingface.co/allenai/OLMoE-1B-7B-0924)
|
| 124 |
+
- [x] [Granite models](https://huggingface.co/collections/ibm-granite/granite-code-models-6624c5cec322e4c148c8b330)
|
| 125 |
+
- [x] [GPT-NeoX](https://github.com/EleutherAI/gpt-neox) + [Pythia](https://github.com/EleutherAI/pythia)
|
| 126 |
+
- [x] [Snowflake-Arctic MoE](https://huggingface.co/collections/Snowflake/arctic-66290090abe542894a5ac520)
|
| 127 |
+
- [x] [Smaug](https://huggingface.co/models?search=Smaug)
|
| 128 |
+
- [x] [Poro 34B](https://huggingface.co/LumiOpen/Poro-34B)
|
| 129 |
+
- [x] [Bitnet b1.58 models](https://huggingface.co/1bitLLM)
|
| 130 |
+
- [x] [Flan T5](https://huggingface.co/models?search=flan-t5)
|
| 131 |
+
- [x] [Open Elm models](https://huggingface.co/collections/apple/openelm-instruct-models-6619ad295d7ae9f868b759ca)
|
| 132 |
+
- [x] [ChatGLM3-6b](https://huggingface.co/THUDM/chatglm3-6b) + [ChatGLM4-9b](https://huggingface.co/THUDM/glm-4-9b) + [GLMEdge-1.5b](https://huggingface.co/THUDM/glm-edge-1.5b-chat) + [GLMEdge-4b](https://huggingface.co/THUDM/glm-edge-4b-chat)
|
| 133 |
+
- [x] [GLM-4-0414](https://huggingface.co/collections/THUDM/glm-4-0414-67f3cbcb34dd9d252707cb2e)
|
| 134 |
+
- [x] [SmolLM](https://huggingface.co/collections/HuggingFaceTB/smollm-6695016cad7167254ce15966)
|
| 135 |
+
- [x] [EXAONE-3.0-7.8B-Instruct](https://huggingface.co/LGAI-EXAONE/EXAONE-3.0-7.8B-Instruct)
|
| 136 |
+
- [x] [FalconMamba Models](https://huggingface.co/collections/tiiuae/falconmamba-7b-66b9a580324dd1598b0f6d4a)
|
| 137 |
+
- [x] [Jais](https://huggingface.co/inceptionai/jais-13b-chat)
|
| 138 |
+
- [x] [Bielik-11B-v2.3](https://huggingface.co/collections/speakleash/bielik-11b-v23-66ee813238d9b526a072408a)
|
| 139 |
+
- [x] [RWKV-7](https://huggingface.co/collections/shoumenchougou/rwkv7-gxx-gguf)
|
| 140 |
+
- [x] [RWKV-6](https://github.com/BlinkDL/RWKV-LM)
|
| 141 |
+
- [x] [QRWKV-6](https://huggingface.co/recursal/QRWKV6-32B-Instruct-Preview-v0.1)
|
| 142 |
+
- [x] [GigaChat-20B-A3B](https://huggingface.co/ai-sage/GigaChat-20B-A3B-instruct)
|
| 143 |
+
- [X] [Trillion-7B-preview](https://huggingface.co/trillionlabs/Trillion-7B-preview)
|
| 144 |
+
- [x] [Ling models](https://huggingface.co/collections/inclusionAI/ling-67c51c85b34a7ea0aba94c32)
|
| 145 |
+
- [x] [Liquid LFM2 models](https://huggingface.co/collections/LiquidAI/lfm2)
|
| 146 |
+
- [x] [Liquid LFM2.5 models](https://huggingface.co/collections/LiquidAI/lfm25)
|
| 147 |
+
- [x] [Liquid Nanos](https://huggingface.co/collections/LiquidAI/liquid-nanos)
|
| 148 |
+
- [x] [Hunyuan models](https://huggingface.co/collections/tencent/hunyuan-dense-model-6890632cda26b19119c9c5e7)
|
| 149 |
+
- [x] [BailingMoeV2 (Ring/Ling 2.0) models](https://huggingface.co/collections/inclusionAI/ling-v2-68bf1dd2fc34c306c1fa6f86)
|
| 150 |
+
- [x] [Mellum models](https://huggingface.co/JetBrains/models?search=mellum)
|
| 151 |
+
|
| 152 |
+
#### Multimodal
|
| 153 |
+
|
| 154 |
+
- [x] [LLaVA 1.5 models](https://huggingface.co/collections/liuhaotian/llava-15-653aac15d994e992e2677a7e), [LLaVA 1.6 models](https://huggingface.co/collections/liuhaotian/llava-16-65b9e40155f60fd046a5ccf2)
|
| 155 |
+
- [x] [BakLLaVA](https://huggingface.co/models?search=SkunkworksAI/Bakllava)
|
| 156 |
+
- [x] [Obsidian](https://huggingface.co/NousResearch/Obsidian-3B-V0.5)
|
| 157 |
+
- [x] [ShareGPT4V](https://huggingface.co/models?search=Lin-Chen/ShareGPT4V)
|
| 158 |
+
- [x] [MobileVLM 1.7B/3B models](https://huggingface.co/models?search=mobileVLM)
|
| 159 |
+
- [x] [Yi-VL](https://huggingface.co/models?search=Yi-VL)
|
| 160 |
+
- [x] [Mini CPM](https://huggingface.co/models?search=MiniCPM)
|
| 161 |
+
- [x] [Moondream](https://huggingface.co/vikhyatk/moondream2)
|
| 162 |
+
- [x] [Bunny](https://github.com/BAAI-DCAI/Bunny)
|
| 163 |
+
- [x] [GLM-EDGE](https://huggingface.co/models?search=glm-edge)
|
| 164 |
+
- [x] [Qwen2-VL](https://huggingface.co/collections/Qwen/qwen2-vl-66cee7455501d7126940800d)
|
| 165 |
+
- [x] [LFM2-VL](https://huggingface.co/collections/LiquidAI/lfm2-vl-68963bbc84a610f7638d5ffa)
|
| 166 |
+
|
| 167 |
+
</details>
|
| 168 |
+
|
| 169 |
+
<details>
|
| 170 |
+
<summary>Bindings</summary>
|
| 171 |
+
|
| 172 |
+
- Python: [ddh0/easy-llama](https://github.com/ddh0/easy-llama)
|
| 173 |
+
- Python: [abetlen/llama-cpp-python](https://github.com/abetlen/llama-cpp-python)
|
| 174 |
+
- Go: [go-skynet/go-llama.cpp](https://github.com/go-skynet/go-llama.cpp)
|
| 175 |
+
- Node.js: [withcatai/node-llama-cpp](https://github.com/withcatai/node-llama-cpp)
|
| 176 |
+
- JS/TS (llama.cpp server client): [lgrammel/modelfusion](https://modelfusion.dev/integration/model-provider/llamacpp)
|
| 177 |
+
- JS/TS (Programmable Prompt Engine CLI): [offline-ai/cli](https://github.com/offline-ai/cli)
|
| 178 |
+
- JavaScript/Wasm (works in browser): [tangledgroup/llama-cpp-wasm](https://github.com/tangledgroup/llama-cpp-wasm)
|
| 179 |
+
- Typescript/Wasm (nicer API, available on npm): [ngxson/wllama](https://github.com/ngxson/wllama)
|
| 180 |
+
- Ruby: [yoshoku/llama_cpp.rb](https://github.com/yoshoku/llama_cpp.rb)
|
| 181 |
+
- Ruby: [docusealco/rllama](https://github.com/docusealco/rllama)
|
| 182 |
+
- Rust (more features): [edgenai/llama_cpp-rs](https://github.com/edgenai/llama_cpp-rs)
|
| 183 |
+
- Rust (nicer API): [mdrokz/rust-llama.cpp](https://github.com/mdrokz/rust-llama.cpp)
|
| 184 |
+
- Rust (more direct bindings): [utilityai/llama-cpp-rs](https://github.com/utilityai/llama-cpp-rs)
|
| 185 |
+
- Rust (automated build from crates.io): [ShelbyJenkins/llm_client](https://github.com/ShelbyJenkins/llm_client)
|
| 186 |
+
- C#/.NET: [SciSharp/LLamaSharp](https://github.com/SciSharp/LLamaSharp)
|
| 187 |
+
- C#/VB.NET (more features - community license): [LM-Kit.NET](https://docs.lm-kit.com/lm-kit-net/index.html)
|
| 188 |
+
- Scala 3: [donderom/llm4s](https://github.com/donderom/llm4s)
|
| 189 |
+
- Clojure: [phronmophobic/llama.clj](https://github.com/phronmophobic/llama.clj)
|
| 190 |
+
- React Native: [mybigday/llama.rn](https://github.com/mybigday/llama.rn)
|
| 191 |
+
- Java: [kherud/java-llama.cpp](https://github.com/kherud/java-llama.cpp)
|
| 192 |
+
- Java: [QuasarByte/llama-cpp-jna](https://github.com/QuasarByte/llama-cpp-jna)
|
| 193 |
+
- Zig: [deins/llama.cpp.zig](https://github.com/Deins/llama.cpp.zig)
|
| 194 |
+
- Flutter/Dart: [netdur/llama_cpp_dart](https://github.com/netdur/llama_cpp_dart)
|
| 195 |
+
- Flutter: [xuegao-tzx/Fllama](https://github.com/xuegao-tzx/Fllama)
|
| 196 |
+
- PHP (API bindings and features built on top of llama.cpp): [distantmagic/resonance](https://github.com/distantmagic/resonance) [(more info)](https://github.com/ggml-org/llama.cpp/pull/6326)
|
| 197 |
+
- Guile Scheme: [guile_llama_cpp](https://savannah.nongnu.org/projects/guile-llama-cpp)
|
| 198 |
+
- Swift [srgtuszy/llama-cpp-swift](https://github.com/srgtuszy/llama-cpp-swift)
|
| 199 |
+
- Swift [ShenghaiWang/SwiftLlama](https://github.com/ShenghaiWang/SwiftLlama)
|
| 200 |
+
- Delphi [Embarcadero/llama-cpp-delphi](https://github.com/Embarcadero/llama-cpp-delphi)
|
| 201 |
+
- Go (no CGo needed): [hybridgroup/yzma](https://github.com/hybridgroup/yzma)
|
| 202 |
+
- Android: [llama.android](/examples/llama.android)
|
| 203 |
+
|
| 204 |
+
</details>
|
| 205 |
+
|
| 206 |
+
<details>
|
| 207 |
+
<summary>UIs</summary>
|
| 208 |
+
|
| 209 |
+
*(to have a project listed here, it should clearly state that it depends on `llama.cpp`)*
|
| 210 |
+
|
| 211 |
+
- [AI Sublime Text plugin](https://github.com/yaroslavyaroslav/OpenAI-sublime-text) (MIT)
|
| 212 |
+
- [BonzAI App](https://apps.apple.com/us/app/bonzai-your-local-ai-agent/id6752847988) (proprietary)
|
| 213 |
+
- [cztomsik/ava](https://github.com/cztomsik/ava) (MIT)
|
| 214 |
+
- [Dot](https://github.com/alexpinel/Dot) (GPL)
|
| 215 |
+
- [eva](https://github.com/ylsdamxssjxxdd/eva) (MIT)
|
| 216 |
+
- [iohub/collama](https://github.com/iohub/coLLaMA) (Apache-2.0)
|
| 217 |
+
- [janhq/jan](https://github.com/janhq/jan) (AGPL)
|
| 218 |
+
- [johnbean393/Sidekick](https://github.com/johnbean393/Sidekick) (MIT)
|
| 219 |
+
- [KanTV](https://github.com/zhouwg/kantv?tab=readme-ov-file) (Apache-2.0)
|
| 220 |
+
- [KodiBot](https://github.com/firatkiral/kodibot) (GPL)
|
| 221 |
+
- [llama.vim](https://github.com/ggml-org/llama.vim) (MIT)
|
| 222 |
+
- [LARS](https://github.com/abgulati/LARS) (AGPL)
|
| 223 |
+
- [Llama Assistant](https://github.com/vietanhdev/llama-assistant) (GPL)
|
| 224 |
+
- [LlamaLib](https://github.com/undreamai/LlamaLib) (Apache-2.0)
|
| 225 |
+
- [LLMFarm](https://github.com/guinmoon/LLMFarm?tab=readme-ov-file) (MIT)
|
| 226 |
+
- [LLMUnity](https://github.com/undreamai/LLMUnity) (MIT)
|
| 227 |
+
- [LMStudio](https://lmstudio.ai/) (proprietary)
|
| 228 |
+
- [LocalAI](https://github.com/mudler/LocalAI) (MIT)
|
| 229 |
+
- [LostRuins/koboldcpp](https://github.com/LostRuins/koboldcpp) (AGPL)
|
| 230 |
+
- [MindMac](https://mindmac.app) (proprietary)
|
| 231 |
+
- [MindWorkAI/AI-Studio](https://github.com/MindWorkAI/AI-Studio) (FSL-1.1-MIT)
|
| 232 |
+
- [Mobile-Artificial-Intelligence/maid](https://github.com/Mobile-Artificial-Intelligence/maid) (MIT)
|
| 233 |
+
- [Mozilla-Ocho/llamafile](https://github.com/Mozilla-Ocho/llamafile) (Apache-2.0)
|
| 234 |
+
- [nat/openplayground](https://github.com/nat/openplayground) (MIT)
|
| 235 |
+
- [nomic-ai/gpt4all](https://github.com/nomic-ai/gpt4all) (MIT)
|
| 236 |
+
- [ollama/ollama](https://github.com/ollama/ollama) (MIT)
|
| 237 |
+
- [oobabooga/text-generation-webui](https://github.com/oobabooga/text-generation-webui) (AGPL)
|
| 238 |
+
- [PocketPal AI](https://github.com/a-ghorbani/pocketpal-ai) (MIT)
|
| 239 |
+
- [psugihara/FreeChat](https://github.com/psugihara/FreeChat) (MIT)
|
| 240 |
+
- [ptsochantaris/emeltal](https://github.com/ptsochantaris/emeltal) (MIT)
|
| 241 |
+
- [pythops/tenere](https://github.com/pythops/tenere) (AGPL)
|
| 242 |
+
- [ramalama](https://github.com/containers/ramalama) (MIT)
|
| 243 |
+
- [semperai/amica](https://github.com/semperai/amica) (MIT)
|
| 244 |
+
- [withcatai/catai](https://github.com/withcatai/catai) (MIT)
|
| 245 |
+
- [Autopen](https://github.com/blackhole89/autopen) (GPL)
|
| 246 |
+
|
| 247 |
+
</details>
|
| 248 |
+
|
| 249 |
+
<details>
|
| 250 |
+
<summary>Tools</summary>
|
| 251 |
+
|
| 252 |
+
- [akx/ggify](https://github.com/akx/ggify) – download PyTorch models from Hugging Face Hub and convert them to GGML
|
| 253 |
+
- [akx/ollama-dl](https://github.com/akx/ollama-dl) – download models from the Ollama library to be used directly with llama.cpp
|
| 254 |
+
- [crashr/gppm](https://github.com/crashr/gppm) – launch llama.cpp instances utilizing NVIDIA Tesla P40 or P100 GPUs with reduced idle power consumption
|
| 255 |
+
- [gpustack/gguf-parser](https://github.com/gpustack/gguf-parser-go/tree/main/cmd/gguf-parser) - review/check the GGUF file and estimate the memory usage
|
| 256 |
+
- [Styled Lines](https://marketplace.unity.com/packages/tools/generative-ai/styled-lines-llama-cpp-model-292902) (proprietary licensed, async wrapper of inference part for game development in Unity3d with pre-built Mobile and Web platform wrappers and a model example)
|
| 257 |
+
- [unslothai/unsloth](https://github.com/unslothai/unsloth) – 🦥 exports/saves fine-tuned and trained models to GGUF (Apache-2.0)
|
| 258 |
+
|
| 259 |
+
</details>
|
| 260 |
+
|
| 261 |
+
<details>
|
| 262 |
+
<summary>Infrastructure</summary>
|
| 263 |
+
|
| 264 |
+
- [Paddler](https://github.com/intentee/paddler) - Open-source LLMOps platform for hosting and scaling AI in your own infrastructure
|
| 265 |
+
- [GPUStack](https://github.com/gpustack/gpustack) - Manage GPU clusters for running LLMs
|
| 266 |
+
- [llama_cpp_canister](https://github.com/onicai/llama_cpp_canister) - llama.cpp as a smart contract on the Internet Computer, using WebAssembly
|
| 267 |
+
- [llama-swap](https://github.com/mostlygeek/llama-swap) - transparent proxy that adds automatic model switching with llama-server
|
| 268 |
+
- [Kalavai](https://github.com/kalavai-net/kalavai-client) - Crowdsource end to end LLM deployment at any scale
|
| 269 |
+
- [llmaz](https://github.com/InftyAI/llmaz) - ☸️ Easy, advanced inference platform for large language models on Kubernetes.
|
| 270 |
+
- [LLMKube](https://github.com/defilantech/llmkube) - Kubernetes operator for llama.cpp with multi-GPU and Apple Silicon Metal
|
| 271 |
+
support"
|
| 272 |
+
</details>
|
| 273 |
+
|
| 274 |
+
<details>
|
| 275 |
+
<summary>Games</summary>
|
| 276 |
+
|
| 277 |
+
- [Lucy's Labyrinth](https://github.com/MorganRO8/Lucys_Labyrinth) - A simple maze game where agents controlled by an AI model will try to trick you.
|
| 278 |
+
|
| 279 |
+
</details>
|
| 280 |
+
|
| 281 |
+
|
| 282 |
+
## Supported backends
|
| 283 |
+
|
| 284 |
+
| Backend | Target devices |
|
| 285 |
+
| --- | --- |
|
| 286 |
+
| [Metal](docs/build.md#metal-build) | Apple Silicon |
|
| 287 |
+
| [BLAS](docs/build.md#blas-build) | All |
|
| 288 |
+
| [BLIS](docs/backend/BLIS.md) | All |
|
| 289 |
+
| [SYCL](docs/backend/SYCL.md) | Intel GPU |
|
| 290 |
+
| [OpenVINO [In Progress]](docs/backend/OPENVINO.md) | Intel CPUs, GPUs, and NPUs |
|
| 291 |
+
| [MUSA](docs/build.md#musa) | Moore Threads GPU |
|
| 292 |
+
| [CUDA](docs/build.md#cuda) | Nvidia GPU |
|
| 293 |
+
| [HIP](docs/build.md#hip) | AMD GPU |
|
| 294 |
+
| [ZenDNN](docs/build.md#zendnn) | AMD CPU |
|
| 295 |
+
| [Vulkan](docs/build.md#vulkan) | GPU |
|
| 296 |
+
| [CANN](docs/build.md#cann) | Ascend NPU |
|
| 297 |
+
| [OpenCL](docs/backend/OPENCL.md) | Adreno GPU |
|
| 298 |
+
| [IBM zDNN](docs/backend/zDNN.md) | IBM Z & LinuxONE |
|
| 299 |
+
| [WebGPU](docs/build.md#webgpu) | All |
|
| 300 |
+
| [RPC](https://github.com/ggml-org/llama.cpp/tree/master/tools/rpc) | All |
|
| 301 |
+
| [Hexagon [In Progress]](docs/backend/snapdragon/README.md) | Snapdragon |
|
| 302 |
+
| [VirtGPU](docs/backend/VirtGPU.md) | VirtGPU APIR |
|
| 303 |
+
|
| 304 |
+
## Obtaining and quantizing models
|
| 305 |
+
|
| 306 |
+
The [Hugging Face](https://huggingface.co) platform hosts a [number of LLMs](https://huggingface.co/models?library=gguf&sort=trending) compatible with `llama.cpp`:
|
| 307 |
+
|
| 308 |
+
- [Trending](https://huggingface.co/models?library=gguf&sort=trending)
|
| 309 |
+
- [LLaMA](https://huggingface.co/models?sort=trending&search=llama+gguf)
|
| 310 |
+
|
| 311 |
+
You can either manually download the GGUF file or directly use any `llama.cpp`-compatible models from [Hugging Face](https://huggingface.co/) or other model hosting sites, by using this CLI argument: `-hf <user>/<model>[:quant]`. For example:
|
| 312 |
+
|
| 313 |
+
```sh
|
| 314 |
+
llama-cli -hf ggml-org/gemma-3-1b-it-GGUF
|
| 315 |
+
```
|
| 316 |
+
|
| 317 |
+
By default, the CLI would download from Hugging Face, you can switch to other options with the environment variable `MODEL_ENDPOINT`. The `MODEL_ENDPOINT` must point to a Hugging Face compatible API endpoint.
|
| 318 |
+
|
| 319 |
+
After downloading a model, use the CLI tools to run it locally - see below.
|
| 320 |
+
|
| 321 |
+
`llama.cpp` requires the model to be stored in the [GGUF](https://github.com/ggml-org/ggml/blob/master/docs/gguf.md) file format. Models in other data formats can be converted to GGUF using the `convert_*.py` Python scripts in this repo.
|
| 322 |
+
|
| 323 |
+
The Hugging Face platform provides a variety of online tools for converting, quantizing and hosting models with `llama.cpp`:
|
| 324 |
+
|
| 325 |
+
- Use the [GGUF-my-repo space](https://huggingface.co/spaces/ggml-org/gguf-my-repo) to convert to GGUF format and quantize model weights to smaller sizes
|
| 326 |
+
- Use the [GGUF-my-LoRA space](https://huggingface.co/spaces/ggml-org/gguf-my-lora) to convert LoRA adapters to GGUF format (more info: https://github.com/ggml-org/llama.cpp/discussions/10123)
|
| 327 |
+
- Use the [GGUF-editor space](https://huggingface.co/spaces/CISCai/gguf-editor) to edit GGUF meta data in the browser (more info: https://github.com/ggml-org/llama.cpp/discussions/9268)
|
| 328 |
+
- Use the [Inference Endpoints](https://ui.endpoints.huggingface.co/) to directly host `llama.cpp` in the cloud (more info: https://github.com/ggml-org/llama.cpp/discussions/9669)
|
| 329 |
+
|
| 330 |
+
To learn more about model quantization, [read this documentation](tools/quantize/README.md)
|
| 331 |
+
|
| 332 |
+
## [`llama-cli`](tools/cli)
|
| 333 |
+
|
| 334 |
+
#### A CLI tool for accessing and experimenting with most of `llama.cpp`'s functionality.
|
| 335 |
+
|
| 336 |
+
- <details open>
|
| 337 |
+
<summary>Run in conversation mode</summary>
|
| 338 |
+
|
| 339 |
+
Models with a built-in chat template will automatically activate conversation mode. If this doesn't occur, you can manually enable it by adding `-cnv` and specifying a suitable chat template with `--chat-template NAME`
|
| 340 |
+
|
| 341 |
+
```bash
|
| 342 |
+
llama-cli -m model.gguf
|
| 343 |
+
|
| 344 |
+
# > hi, who are you?
|
| 345 |
+
# Hi there! I'm your helpful assistant! I'm an AI-powered chatbot designed to assist and provide information to users like you. I'm here to help answer your questions, provide guidance, and offer support on a wide range of topics. I'm a friendly and knowledgeable AI, and I'm always happy to help with anything you need. What's on your mind, and how can I assist you today?
|
| 346 |
+
#
|
| 347 |
+
# > what is 1+1?
|
| 348 |
+
# Easy peasy! The answer to 1+1 is... 2!
|
| 349 |
+
```
|
| 350 |
+
|
| 351 |
+
</details>
|
| 352 |
+
|
| 353 |
+
- <details>
|
| 354 |
+
<summary>Run in conversation mode with custom chat template</summary>
|
| 355 |
+
|
| 356 |
+
```bash
|
| 357 |
+
# use the "chatml" template (use -h to see the list of supported templates)
|
| 358 |
+
llama-cli -m model.gguf -cnv --chat-template chatml
|
| 359 |
+
|
| 360 |
+
# use a custom template
|
| 361 |
+
llama-cli -m model.gguf -cnv --in-prefix 'User: ' --reverse-prompt 'User:'
|
| 362 |
+
```
|
| 363 |
+
|
| 364 |
+
</details>
|
| 365 |
+
|
| 366 |
+
- <details>
|
| 367 |
+
<summary>Constrain the output with a custom grammar</summary>
|
| 368 |
+
|
| 369 |
+
```bash
|
| 370 |
+
llama-cli -m model.gguf -n 256 --grammar-file grammars/json.gbnf -p 'Request: schedule a call at 8pm; Command:'
|
| 371 |
+
|
| 372 |
+
# {"appointmentTime": "8pm", "appointmentDetails": "schedule a a call"}
|
| 373 |
+
```
|
| 374 |
+
|
| 375 |
+
The [grammars/](grammars/) folder contains a handful of sample grammars. To write your own, check out the [GBNF Guide](grammars/README.md).
|
| 376 |
+
|
| 377 |
+
For authoring more complex JSON grammars, check out https://grammar.intrinsiclabs.ai/
|
| 378 |
+
|
| 379 |
+
</details>
|
| 380 |
+
|
| 381 |
+
|
| 382 |
+
## [`llama-server`](tools/server)
|
| 383 |
+
|
| 384 |
+
#### A lightweight, [OpenAI API](https://github.com/openai/openai-openapi) compatible, HTTP server for serving LLMs.
|
| 385 |
+
|
| 386 |
+
- <details open>
|
| 387 |
+
<summary>Start a local HTTP server with default configuration on port 8080</summary>
|
| 388 |
+
|
| 389 |
+
```bash
|
| 390 |
+
llama-server -m model.gguf --port 8080
|
| 391 |
+
|
| 392 |
+
# Basic web UI can be accessed via browser: http://localhost:8080
|
| 393 |
+
# Chat completion endpoint: http://localhost:8080/v1/chat/completions
|
| 394 |
+
```
|
| 395 |
+
|
| 396 |
+
</details>
|
| 397 |
+
|
| 398 |
+
- <details>
|
| 399 |
+
<summary>Support multiple-users and parallel decoding</summary>
|
| 400 |
+
|
| 401 |
+
```bash
|
| 402 |
+
# up to 4 concurrent requests, each with 4096 max context
|
| 403 |
+
llama-server -m model.gguf -c 16384 -np 4
|
| 404 |
+
```
|
| 405 |
+
|
| 406 |
+
</details>
|
| 407 |
+
|
| 408 |
+
- <details>
|
| 409 |
+
<summary>Enable speculative decoding</summary>
|
| 410 |
+
|
| 411 |
+
```bash
|
| 412 |
+
# the draft.gguf model should be a small variant of the target model.gguf
|
| 413 |
+
llama-server -m model.gguf -md draft.gguf
|
| 414 |
+
```
|
| 415 |
+
|
| 416 |
+
</details>
|
| 417 |
+
|
| 418 |
+
- <details>
|
| 419 |
+
<summary>Serve an embedding model</summary>
|
| 420 |
+
|
| 421 |
+
```bash
|
| 422 |
+
# use the /embedding endpoint
|
| 423 |
+
llama-server -m model.gguf --embedding --pooling cls -ub 8192
|
| 424 |
+
```
|
| 425 |
+
|
| 426 |
+
</details>
|
| 427 |
+
|
| 428 |
+
- <details>
|
| 429 |
+
<summary>Serve a reranking model</summary>
|
| 430 |
+
|
| 431 |
+
```bash
|
| 432 |
+
# use the /reranking endpoint
|
| 433 |
+
llama-server -m model.gguf --reranking
|
| 434 |
+
```
|
| 435 |
+
|
| 436 |
+
</details>
|
| 437 |
+
|
| 438 |
+
- <details>
|
| 439 |
+
<summary>Constrain all outputs with a grammar</summary>
|
| 440 |
+
|
| 441 |
+
```bash
|
| 442 |
+
# custom grammar
|
| 443 |
+
llama-server -m model.gguf --grammar-file grammar.gbnf
|
| 444 |
+
|
| 445 |
+
# JSON
|
| 446 |
+
llama-server -m model.gguf --grammar-file grammars/json.gbnf
|
| 447 |
+
```
|
| 448 |
+
|
| 449 |
+
</details>
|
| 450 |
+
|
| 451 |
+
|
| 452 |
+
## [`llama-perplexity`](tools/perplexity)
|
| 453 |
+
|
| 454 |
+
#### A tool for measuring the [perplexity](tools/perplexity/README.md) [^1] (and other quality metrics) of a model over a given text.
|
| 455 |
+
|
| 456 |
+
- <details open>
|
| 457 |
+
<summary>Measure the perplexity over a text file</summary>
|
| 458 |
+
|
| 459 |
+
```bash
|
| 460 |
+
llama-perplexity -m model.gguf -f file.txt
|
| 461 |
+
|
| 462 |
+
# [1]15.2701,[2]5.4007,[3]5.3073,[4]6.2965,[5]5.8940,[6]5.6096,[7]5.7942,[8]4.9297, ...
|
| 463 |
+
# Final estimate: PPL = 5.4007 +/- 0.67339
|
| 464 |
+
```
|
| 465 |
+
|
| 466 |
+
</details>
|
| 467 |
+
|
| 468 |
+
- <details>
|
| 469 |
+
<summary>Measure KL divergence</summary>
|
| 470 |
+
|
| 471 |
+
```bash
|
| 472 |
+
# TODO
|
| 473 |
+
```
|
| 474 |
+
|
| 475 |
+
</details>
|
| 476 |
+
|
| 477 |
+
[^1]: [https://huggingface.co/docs/transformers/perplexity](https://huggingface.co/docs/transformers/perplexity)
|
| 478 |
+
|
| 479 |
+
## [`llama-bench`](tools/llama-bench)
|
| 480 |
+
|
| 481 |
+
#### Benchmark the performance of the inference for various parameters.
|
| 482 |
+
|
| 483 |
+
- <details open>
|
| 484 |
+
<summary>Run default benchmark</summary>
|
| 485 |
+
|
| 486 |
+
```bash
|
| 487 |
+
llama-bench -m model.gguf
|
| 488 |
+
|
| 489 |
+
# Output:
|
| 490 |
+
# | model | size | params | backend | threads | test | t/s |
|
| 491 |
+
# | ------------------- | ---------: | ---------: | ---------- | ------: | ------------: | -------------------: |
|
| 492 |
+
# | qwen2 1.5B Q4_0 | 885.97 MiB | 1.54 B | Metal,BLAS | 16 | pp512 | 5765.41 ± 20.55 |
|
| 493 |
+
# | qwen2 1.5B Q4_0 | 885.97 MiB | 1.54 B | Metal,BLAS | 16 | tg128 | 197.71 ± 0.81 |
|
| 494 |
+
#
|
| 495 |
+
# build: 3e0ba0e60 (4229)
|
| 496 |
+
```
|
| 497 |
+
|
| 498 |
+
</details>
|
| 499 |
+
|
| 500 |
+
## [`llama-simple`](examples/simple)
|
| 501 |
+
|
| 502 |
+
#### A minimal example for implementing apps with `llama.cpp`. Useful for developers.
|
| 503 |
+
|
| 504 |
+
- <details>
|
| 505 |
+
<summary>Basic text completion</summary>
|
| 506 |
+
|
| 507 |
+
```bash
|
| 508 |
+
llama-simple -m model.gguf
|
| 509 |
+
|
| 510 |
+
# Hello my name is Kaitlyn and I am a 16 year old girl. I am a junior in high school and I am currently taking a class called "The Art of
|
| 511 |
+
```
|
| 512 |
+
|
| 513 |
+
</details>
|
| 514 |
+
|
| 515 |
+
|
| 516 |
+
## Contributing
|
| 517 |
+
|
| 518 |
+
- Contributors can open PRs
|
| 519 |
+
- Collaborators will be invited based on contributions
|
| 520 |
+
- Maintainers can push to branches in the `llama.cpp` repo and merge PRs into the `master` branch
|
| 521 |
+
- Any help with managing issues, PRs and projects is very appreciated!
|
| 522 |
+
- See [good first issues](https://github.com/ggml-org/llama.cpp/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) for tasks suitable for first contributions
|
| 523 |
+
- Read the [CONTRIBUTING.md](CONTRIBUTING.md) for more information
|
| 524 |
+
- Make sure to read this: [Inference at the edge](https://github.com/ggml-org/llama.cpp/discussions/205)
|
| 525 |
+
- A bit of backstory for those who are interested: [Changelog podcast](https://changelog.com/podcast/532)
|
| 526 |
+
|
| 527 |
+
## Other documentation
|
| 528 |
+
|
| 529 |
+
- [cli](tools/cli/README.md)
|
| 530 |
+
- [completion](tools/completion/README.md)
|
| 531 |
+
- [server](tools/server/README.md)
|
| 532 |
+
- [GBNF grammars](grammars/README.md)
|
| 533 |
+
|
| 534 |
+
#### Development documentation
|
| 535 |
+
|
| 536 |
+
- [How to build](docs/build.md)
|
| 537 |
+
- [Running on Docker](docs/docker.md)
|
| 538 |
+
- [Build on Android](docs/android.md)
|
| 539 |
+
- [Multi-GPU usage](docs/multi-gpu.md)
|
| 540 |
+
- [Performance troubleshooting](docs/development/token_generation_performance_tips.md)
|
| 541 |
+
- [GGML tips & tricks](https://github.com/ggml-org/llama.cpp/wiki/GGML-Tips-&-Tricks)
|
| 542 |
+
|
| 543 |
+
#### Seminal papers and background on the models
|
| 544 |
+
|
| 545 |
+
If your issue is with model generation quality, then please at least scan the following links and papers to understand the limitations of LLaMA models. This is especially important when choosing an appropriate model size and appreciating both the significant and subtle differences between LLaMA models and ChatGPT:
|
| 546 |
+
- LLaMA:
|
| 547 |
+
- [Introducing LLaMA: A foundational, 65-billion-parameter large language model](https://ai.facebook.com/blog/large-language-model-llama-meta-ai/)
|
| 548 |
+
- [LLaMA: Open and Efficient Foundation Language Models](https://arxiv.org/abs/2302.13971)
|
| 549 |
+
- GPT-3
|
| 550 |
+
- [Language Models are Few-Shot Learners](https://arxiv.org/abs/2005.14165)
|
| 551 |
+
- GPT-3.5 / InstructGPT / ChatGPT:
|
| 552 |
+
- [Aligning language models to follow instructions](https://openai.com/research/instruction-following)
|
| 553 |
+
- [Training language models to follow instructions with human feedback](https://arxiv.org/abs/2203.02155)
|
| 554 |
+
|
| 555 |
+
## XCFramework
|
| 556 |
+
The XCFramework is a precompiled version of the library for iOS, visionOS, tvOS,
|
| 557 |
+
and macOS. It can be used in Swift projects without the need to compile the
|
| 558 |
+
library from source. For example:
|
| 559 |
+
```swift
|
| 560 |
+
// swift-tools-version: 5.10
|
| 561 |
+
// The swift-tools-version declares the minimum version of Swift required to build this package.
|
| 562 |
+
|
| 563 |
+
import PackageDescription
|
| 564 |
+
|
| 565 |
+
let package = Package(
|
| 566 |
+
name: "MyLlamaPackage",
|
| 567 |
+
targets: [
|
| 568 |
+
.executableTarget(
|
| 569 |
+
name: "MyLlamaPackage",
|
| 570 |
+
dependencies: [
|
| 571 |
+
"LlamaFramework"
|
| 572 |
+
]),
|
| 573 |
+
.binaryTarget(
|
| 574 |
+
name: "LlamaFramework",
|
| 575 |
+
url: "https://github.com/ggml-org/llama.cpp/releases/download/b5046/llama-b5046-xcframework.zip",
|
| 576 |
+
checksum: "c19be78b5f00d8d29a25da41042cb7afa094cbf6280a225abe614b03b20029ab"
|
| 577 |
+
)
|
| 578 |
+
]
|
| 579 |
+
)
|
| 580 |
+
```
|
| 581 |
+
The above example is using an intermediate build `b5046` of the library. This can be modified
|
| 582 |
+
to use a different version by changing the URL and checksum.
|
| 583 |
+
|
| 584 |
+
## Completions
|
| 585 |
+
Command-line completion is available for some environments.
|
| 586 |
+
|
| 587 |
+
#### Bash Completion
|
| 588 |
+
```bash
|
| 589 |
+
$ build/bin/llama-cli --completion-bash > ~/.llama-completion.bash
|
| 590 |
+
$ source ~/.llama-completion.bash
|
| 591 |
+
```
|
| 592 |
+
Optionally this can be added to your `.bashrc` or `.bash_profile` to load it
|
| 593 |
+
automatically. For example:
|
| 594 |
+
```console
|
| 595 |
+
$ echo "source ~/.llama-completion.bash" >> ~/.bashrc
|
| 596 |
+
```
|
| 597 |
+
|
| 598 |
+
## Dependencies
|
| 599 |
+
|
| 600 |
+
- [yhirose/cpp-httplib](https://github.com/yhirose/cpp-httplib) - Single-header HTTP server, used by `llama-server` - MIT license
|
| 601 |
+
- [stb-image](https://github.com/nothings/stb) - Single-header image format decoder, used by multimodal subsystem - Public domain
|
| 602 |
+
- [nlohmann/json](https://github.com/nlohmann/json) - Single-header JSON library, used by various tools/examples - MIT License
|
| 603 |
+
- [miniaudio.h](https://github.com/mackron/miniaudio) - Single-header audio format decoder, used by multimodal subsystem - Public domain
|
| 604 |
+
- [subprocess.h](https://github.com/sheredom/subprocess.h) - Single-header process launching solution for C and C++ - Public domain
|
llama.cpp/SECURITY.md
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Security Policy
|
| 2 |
+
|
| 3 |
+
- [**Reporting a vulnerability**](#reporting-a-vulnerability)
|
| 4 |
+
- [**Requirements**](#requirements)
|
| 5 |
+
- [**Covered Topics**](#covered-topics)
|
| 6 |
+
- [**Using llama.cpp securely**](#using-llamacpp-securely)
|
| 7 |
+
- [Untrusted models](#untrusted-models)
|
| 8 |
+
- [Untrusted inputs](#untrusted-inputs)
|
| 9 |
+
- [Data privacy](#data-privacy)
|
| 10 |
+
- [Untrusted environments or networks](#untrusted-environments-or-networks)
|
| 11 |
+
- [Multi-Tenant environments](#multi-tenant-environments)
|
| 12 |
+
|
| 13 |
+
## Reporting a vulnerability
|
| 14 |
+
|
| 15 |
+
> [!IMPORTANT]
|
| 16 |
+
> The private security disclosure program is disabled until further notice. Please submit patches with fixes directly to the repo as public PRs. Emails will be ignored.
|
| 17 |
+
|
| 18 |
+
If you have discovered a security vulnerability in this project that falls inside the [covered topics](#covered-topics), please report it privately. **Do not disclose it as a public issue.** This gives us time to work with you to fix the issue before public exposure, reducing the chance that the exploit will be used before a patch is released.
|
| 19 |
+
|
| 20 |
+
Please disclose it as a private [security advisory](https://github.com/ggml-org/llama.cpp/security/advisories/new).
|
| 21 |
+
|
| 22 |
+
A team of volunteers on a reasonable-effort basis maintains this project. As such, please give us at least 90 days to work on a fix before public exposure.
|
| 23 |
+
|
| 24 |
+
### Requirements
|
| 25 |
+
|
| 26 |
+
Before submitting your report, ensure you meet the following requirements:
|
| 27 |
+
|
| 28 |
+
- You have read this policy and fully understand it.
|
| 29 |
+
- AI is only permitted in an assistive capacity as stated in [AGENTS.md](AGENTS.md). We do not accept reports that are written exclusively by AI.
|
| 30 |
+
- Your report must include a working Proof-of-Concept in the form of a script and/or attached files.
|
| 31 |
+
|
| 32 |
+
Maintainers reserve the right to close the report if these requirements are not fulfilled.
|
| 33 |
+
|
| 34 |
+
### Covered Topics
|
| 35 |
+
|
| 36 |
+
Only vulnerabilities that fall within these parts of the project are considered valid. For problems falling outside of this list, please report them as issues.
|
| 37 |
+
|
| 38 |
+
- `src/**/*`
|
| 39 |
+
- `ggml/**/*`
|
| 40 |
+
- `gguf-py/**/*`
|
| 41 |
+
- `tools/server/*`, **excluding** the following topics:
|
| 42 |
+
- Web UI
|
| 43 |
+
- Features marked as experimental
|
| 44 |
+
- Features not recommended for use in untrusted environments (e.g., router, MCP)
|
| 45 |
+
- Bugs that can lead to Denial-of-Service attack
|
| 46 |
+
|
| 47 |
+
Note that none of the topics under [Using llama.cpp securely](#using-llamacpp-securely) are considered vulnerabilities in LLaMA C++.
|
| 48 |
+
|
| 49 |
+
For vulnerabilities that fall within the `vendor` directory, please report them directly to the third-party project.
|
| 50 |
+
|
| 51 |
+
## Using llama.cpp securely
|
| 52 |
+
|
| 53 |
+
### Untrusted models
|
| 54 |
+
Be careful when running untrusted models. This classification includes models created by unknown developers or utilizing data obtained from unknown sources.
|
| 55 |
+
|
| 56 |
+
*Always execute untrusted models within a secure, isolated environment such as a sandbox* (e.g., containers, virtual machines). This helps protect your system from potentially malicious code.
|
| 57 |
+
|
| 58 |
+
> [!NOTE]
|
| 59 |
+
> The trustworthiness of a model is not binary. You must always determine the proper level of caution depending on the specific model and how it matches your use case and risk tolerance.
|
| 60 |
+
|
| 61 |
+
### Untrusted inputs
|
| 62 |
+
|
| 63 |
+
Some models accept various input formats (text, images, audio, etc.). The libraries converting these inputs have varying security levels, so it's crucial to isolate the model and carefully pre-process inputs to mitigate script injection risks.
|
| 64 |
+
|
| 65 |
+
For maximum security when handling untrusted inputs, you may need to employ the following:
|
| 66 |
+
|
| 67 |
+
* Sandboxing: Isolate the environment where the inference happens.
|
| 68 |
+
* Pre-analysis: Check how the model performs by default when exposed to prompt injection (e.g. using [fuzzing for prompt injection](https://github.com/FonduAI/awesome-prompt-injection?tab=readme-ov-file#tools)). This will give you leads on how hard you will have to work on the next topics.
|
| 69 |
+
* Updates: Keep both LLaMA C++ and your libraries updated with the latest security patches.
|
| 70 |
+
* Input Sanitation: Before feeding data to the model, sanitize inputs rigorously. This involves techniques such as:
|
| 71 |
+
* Validation: Enforce strict rules on allowed characters and data types.
|
| 72 |
+
* Filtering: Remove potentially malicious scripts or code fragments.
|
| 73 |
+
* Encoding: Convert special characters into safe representations.
|
| 74 |
+
* Verification: Run tooling that identifies potential script injections (e.g. [models that detect prompt injection attempts](https://python.langchain.com/docs/guides/safety/hugging_face_prompt_injection)).
|
| 75 |
+
|
| 76 |
+
### Data privacy
|
| 77 |
+
|
| 78 |
+
To protect sensitive data from potential leaks or unauthorized access, it is crucial to sandbox the model execution. This means running the model in a secure, isolated environment, which helps mitigate many attack vectors.
|
| 79 |
+
|
| 80 |
+
### Untrusted environments or networks
|
| 81 |
+
|
| 82 |
+
If you can't run your models in a secure and isolated environment or if it must be exposed to an untrusted network, make sure to take the following security precautions:
|
| 83 |
+
* Do not use the RPC backend, [ggml-rpc-server](https://github.com/ggml-org/llama.cpp/tree/master/tools/rpc) and [llama-server](https://github.com/ggml-org/llama.cpp/tree/master/tools/server) functionality (see https://github.com/ggml-org/llama.cpp/pull/13061).
|
| 84 |
+
* Confirm the hash of any downloaded artifact (e.g. pre-trained model weights) matches a known-good value.
|
| 85 |
+
* Encrypt your data if sending it over the network.
|
| 86 |
+
|
| 87 |
+
### Multi-Tenant environments
|
| 88 |
+
|
| 89 |
+
If you intend to run multiple models in parallel with shared memory, it is your responsibility to ensure the models do not interact or access each other's data. The primary areas of concern are tenant isolation, resource allocation, model sharing and hardware attacks.
|
| 90 |
+
|
| 91 |
+
1. Tenant Isolation: Models should run separately with strong isolation methods to prevent unwanted data access. Separating networks is crucial for isolation, as it prevents unauthorized access to data or models and malicious users from sending graphs to execute under another tenant's identity.
|
| 92 |
+
|
| 93 |
+
2. Resource Allocation: A denial of service caused by one model can impact the overall system health. Implement safeguards like rate limits, access controls, and health monitoring.
|
| 94 |
+
|
| 95 |
+
3. Model Sharing: In a multitenant model sharing design, tenants and users must understand the security risks of running code provided by others. Since there are no reliable methods to detect malicious models, sandboxing the model execution is the recommended approach to mitigate the risk.
|
| 96 |
+
|
| 97 |
+
4. Hardware Attacks: GPUs or TPUs can also be attacked. [Researches](https://scholar.google.com/scholar?q=gpu+side+channel) has shown that side channel attacks on GPUs are possible, which can make data leak from other models or processes running on the same system at the same time.
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/im2col_3d.comp
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
|
| 3 |
+
#extension GL_EXT_shader_16bit_storage : require
|
| 4 |
+
#extension GL_EXT_control_flow_attributes : require
|
| 5 |
+
#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
|
| 6 |
+
|
| 7 |
+
#include "types.glsl"
|
| 8 |
+
|
| 9 |
+
layout (push_constant) uniform parameter
|
| 10 |
+
{
|
| 11 |
+
BDA_STORAGE_T dst_addr;
|
| 12 |
+
uint32_t nb10;
|
| 13 |
+
uint32_t nb11;
|
| 14 |
+
uint32_t nb12;
|
| 15 |
+
uint32_t nb13;
|
| 16 |
+
uint32_t s0;
|
| 17 |
+
uint32_t s1;
|
| 18 |
+
uint32_t s2;
|
| 19 |
+
uint32_t p0;
|
| 20 |
+
uint32_t p1;
|
| 21 |
+
uint32_t p2;
|
| 22 |
+
uint32_t d0;
|
| 23 |
+
uint32_t d1;
|
| 24 |
+
uint32_t d2;
|
| 25 |
+
uint32_t IW;
|
| 26 |
+
uint32_t IH;
|
| 27 |
+
uint32_t ID;
|
| 28 |
+
uint32_t IC;
|
| 29 |
+
uint32_t KW;
|
| 30 |
+
uint32_t OH;
|
| 31 |
+
uint32_t KD_KH_KW;
|
| 32 |
+
uint32_t KH_KW;
|
| 33 |
+
uint32_t IC_KD_KH_KW;
|
| 34 |
+
uint32_t N_OD_OH;
|
| 35 |
+
uint32_t OD_OH;
|
| 36 |
+
uint32_t OD_OH_OW_IC_KD_KH_KW;
|
| 37 |
+
uint32_t OH_OW_IC_KD_KH_KW;
|
| 38 |
+
uint32_t OW_IC_KD_KH_KW;
|
| 39 |
+
uint32_t misalign_offsets;
|
| 40 |
+
} p;
|
| 41 |
+
|
| 42 |
+
uint get_aoffset() { return p.misalign_offsets >> 16; }
|
| 43 |
+
uint get_doffset() { return p.misalign_offsets & 0xFFFF; }
|
| 44 |
+
|
| 45 |
+
layout(constant_id = 0) const uint BLOCK_SIZE = 32;
|
| 46 |
+
|
| 47 |
+
layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
|
| 48 |
+
|
| 49 |
+
layout (binding = 0) readonly buffer X {A_TYPE data_a[];};
|
| 50 |
+
layout (binding = 1) writeonly buffer D {D_TYPE data_d[];};
|
| 51 |
+
|
| 52 |
+
#if BDA
|
| 53 |
+
layout (buffer_reference) buffer D_ptr {D_TYPE d;};
|
| 54 |
+
#endif
|
| 55 |
+
|
| 56 |
+
void main() {
|
| 57 |
+
const uint32_t i = gl_GlobalInvocationID.x;
|
| 58 |
+
|
| 59 |
+
uint32_t nb10 = p.nb10;
|
| 60 |
+
uint32_t nb11 = p.nb11;
|
| 61 |
+
uint32_t nb12 = p.nb12;
|
| 62 |
+
uint32_t nb13 = p.nb13;
|
| 63 |
+
uint32_t s0 = p.s0;
|
| 64 |
+
uint32_t s1 = p.s1;
|
| 65 |
+
uint32_t s2 = p.s2;
|
| 66 |
+
uint32_t p0 = p.p0;
|
| 67 |
+
uint32_t p1 = p.p1;
|
| 68 |
+
uint32_t p2 = p.p2;
|
| 69 |
+
uint32_t d0 = p.d0;
|
| 70 |
+
uint32_t d1 = p.d1;
|
| 71 |
+
uint32_t d2 = p.d2;
|
| 72 |
+
uint32_t IW = p.IW;
|
| 73 |
+
uint32_t IH = p.IH;
|
| 74 |
+
uint32_t ID = p.ID;
|
| 75 |
+
uint32_t IC = p.IC;
|
| 76 |
+
uint32_t KW = p.KW;
|
| 77 |
+
uint32_t OH = p.OH;
|
| 78 |
+
uint32_t KD_KH_KW = p.KD_KH_KW;
|
| 79 |
+
uint32_t KH_KW = p.KH_KW;
|
| 80 |
+
uint32_t IC_KD_KH_KW = p.IC_KD_KH_KW;
|
| 81 |
+
uint32_t N_OD_OH = p.N_OD_OH;
|
| 82 |
+
uint32_t OD_OH = p.OD_OH;
|
| 83 |
+
uint32_t OD_OH_OW_IC_KD_KH_KW = p.OD_OH_OW_IC_KD_KH_KW;
|
| 84 |
+
uint32_t OH_OW_IC_KD_KH_KW = p.OH_OW_IC_KD_KH_KW;
|
| 85 |
+
uint32_t OW_IC_KD_KH_KW = p.OW_IC_KD_KH_KW;
|
| 86 |
+
|
| 87 |
+
if (i >= IC_KD_KH_KW) {
|
| 88 |
+
return;
|
| 89 |
+
}
|
| 90 |
+
|
| 91 |
+
const uint32_t iic = i / KD_KH_KW;
|
| 92 |
+
const uint32_t ikd = (i - iic * KD_KH_KW) / KH_KW;
|
| 93 |
+
const uint32_t ikh = (i - iic * KD_KH_KW - ikd * KH_KW) / KW;
|
| 94 |
+
const uint32_t ikw = i % KW;
|
| 95 |
+
|
| 96 |
+
const uint32_t iow = gl_GlobalInvocationID.y;
|
| 97 |
+
for (uint32_t iz = gl_GlobalInvocationID.z; iz < N_OD_OH; iz += gl_NumWorkGroups.z) {
|
| 98 |
+
const uint32_t in_ = iz / OD_OH;
|
| 99 |
+
const uint32_t iod = (iz - in_*OD_OH) / OH;
|
| 100 |
+
const uint32_t ioh = iz % OH;
|
| 101 |
+
|
| 102 |
+
const uint32_t iiw = iow * s0 + ikw * d0 - p0;
|
| 103 |
+
const uint32_t iih = ioh * s1 + ikh * d1 - p1;
|
| 104 |
+
const uint32_t iid = iod * s2 + ikd * d2 - p2;
|
| 105 |
+
|
| 106 |
+
const BDA_OFFSET_T offset_dst = BDA_OFFSET_T(in_)*OD_OH_OW_IC_KD_KH_KW + BDA_OFFSET_T(iod)*OH_OW_IC_KD_KH_KW + BDA_OFFSET_T(ioh)*OW_IC_KD_KH_KW + BDA_OFFSET_T(iow)*IC_KD_KH_KW + iic*KD_KH_KW + ikd * KH_KW + ikh*KW + ikw;
|
| 107 |
+
|
| 108 |
+
const uint32_t offset_src = (in_*IC + iic)*nb13 + iid*nb12 + iih*nb11 + iiw*nb10;
|
| 109 |
+
#if BDA
|
| 110 |
+
D_ptr dst_addr = D_ptr(p.dst_addr + D_SIZE * offset_dst);
|
| 111 |
+
if (iih >= IH || iiw >= IW || iid >= ID) {
|
| 112 |
+
dst_addr.d = D_TYPE(0.0f);
|
| 113 |
+
} else {
|
| 114 |
+
dst_addr.d = D_TYPE(data_a[offset_src + get_aoffset()]);
|
| 115 |
+
}
|
| 116 |
+
#else
|
| 117 |
+
if (iih >= IH || iiw >= IW || iid >= ID) {
|
| 118 |
+
data_d[offset_dst + get_doffset()] = D_TYPE(0.0f);
|
| 119 |
+
} else {
|
| 120 |
+
data_d[offset_dst + get_doffset()] = D_TYPE(data_a[offset_src + get_aoffset()]);
|
| 121 |
+
}
|
| 122 |
+
#endif
|
| 123 |
+
}
|
| 124 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/l2_norm.comp
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
|
| 3 |
+
#include "generic_unary_head.glsl"
|
| 4 |
+
#include "types.glsl"
|
| 5 |
+
|
| 6 |
+
#extension GL_EXT_control_flow_attributes : enable
|
| 7 |
+
#define BLOCK_SIZE 512
|
| 8 |
+
|
| 9 |
+
layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in;
|
| 10 |
+
|
| 11 |
+
shared FLOAT_TYPE sum[BLOCK_SIZE];
|
| 12 |
+
|
| 13 |
+
void main() {
|
| 14 |
+
const uint row = gl_WorkGroupID.z * 262144 + gl_WorkGroupID.y * 512 + gl_WorkGroupID.x;
|
| 15 |
+
const uint tid = gl_LocalInvocationID.x;
|
| 16 |
+
|
| 17 |
+
const uint a_base = get_aoffset() + src0_idx(row * p.ne00);
|
| 18 |
+
const uint d_base = get_doffset() + dst_idx(row * p.ne10);
|
| 19 |
+
|
| 20 |
+
sum[tid] = FLOAT_TYPE(0.0f); // partial sum for thread in warp
|
| 21 |
+
|
| 22 |
+
[[unroll]] for (uint i0 = tid; i0 < p.ne00; i0 += BLOCK_SIZE) {
|
| 23 |
+
const FLOAT_TYPE xi = FLOAT_TYPE(data_a[a_base + i0*p.nb00]);
|
| 24 |
+
sum[tid] += xi * xi;
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
// sum up partial sums and write back result
|
| 28 |
+
barrier();
|
| 29 |
+
[[unroll]] for (int s = BLOCK_SIZE / 2; s > 0; s >>= 1) {
|
| 30 |
+
if (tid < s) {
|
| 31 |
+
sum[tid] += sum[tid + s];
|
| 32 |
+
}
|
| 33 |
+
barrier();
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
const FLOAT_TYPE scale = 1.0f / max(sqrt(sum[0]), FLOAT_TYPE(p.param1));
|
| 37 |
+
|
| 38 |
+
[[unroll]] for (uint i0 = tid; i0 < p.ne00; i0 += BLOCK_SIZE) {
|
| 39 |
+
data_d[d_base + i0*p.nb10] = D_TYPE(scale * FLOAT_TYPE(data_a[a_base + i0*p.nb00]));
|
| 40 |
+
}
|
| 41 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/log.comp
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
|
| 3 |
+
#include "types.glsl"
|
| 4 |
+
#include "generic_unary_head.glsl"
|
| 5 |
+
|
| 6 |
+
layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
|
| 7 |
+
|
| 8 |
+
void main() {
|
| 9 |
+
const uint idx = get_idx();
|
| 10 |
+
|
| 11 |
+
if (idx >= p.ne) {
|
| 12 |
+
return;
|
| 13 |
+
}
|
| 14 |
+
|
| 15 |
+
const float val = float(data_a[get_aoffset() + src0_idx(idx)]);
|
| 16 |
+
data_d[get_doffset() + dst_idx(idx)] = D_TYPE(log(val));
|
| 17 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul.comp
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
|
| 3 |
+
#include "types.glsl"
|
| 4 |
+
#include "generic_binary_head.glsl"
|
| 5 |
+
|
| 6 |
+
const uint num_threads = 256;
|
| 7 |
+
|
| 8 |
+
layout(local_size_x = num_threads, local_size_y = 1, local_size_z = 1) in;
|
| 9 |
+
|
| 10 |
+
void main() {
|
| 11 |
+
uint idx = get_idx();
|
| 12 |
+
|
| 13 |
+
// num_threads * num_iter must equal 512, to match the wg_denoms and get_idx calculation
|
| 14 |
+
const uint num_iter = 2;
|
| 15 |
+
|
| 16 |
+
[[unroll]] for (uint i = 0; i < num_iter; ++i) {
|
| 17 |
+
if (idx >= p.ne) {
|
| 18 |
+
continue;
|
| 19 |
+
}
|
| 20 |
+
uint i00, i01, i02, i03;
|
| 21 |
+
get_indices(idx, i00, i01, i02, i03);
|
| 22 |
+
|
| 23 |
+
data_d[get_doffset() + dst_idx(i00, i01, i02, i03)] = D_TYPE(FLOAT_TYPE(data_a[get_aoffset() + src0_idx(i00, i01, i02, i03)]) * FLOAT_TYPE(data_b[get_boffset() + src1_idx(i00, i01, i02, i03)]));
|
| 24 |
+
|
| 25 |
+
idx += num_threads;
|
| 26 |
+
}
|
| 27 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_split_k_reduce.comp
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
|
| 3 |
+
#extension GL_EXT_control_flow_attributes : enable
|
| 4 |
+
|
| 5 |
+
layout(local_size_x = 256, local_size_y = 1, local_size_z = 1) in;
|
| 6 |
+
|
| 7 |
+
layout (binding = 0) readonly buffer A {float data_a[];};
|
| 8 |
+
layout (binding = 0) readonly buffer A4 {vec4 data_a4[];};
|
| 9 |
+
layout (binding = 1) writeonly buffer D {float data_d[];};
|
| 10 |
+
layout (binding = 1) writeonly buffer D4 {vec4 data_d4[];};
|
| 11 |
+
|
| 12 |
+
layout (push_constant) uniform parameter {
|
| 13 |
+
uint ne;
|
| 14 |
+
uint k_num;
|
| 15 |
+
} p;
|
| 16 |
+
|
| 17 |
+
void main() {
|
| 18 |
+
// Each invocation handles four consecutive components
|
| 19 |
+
const uint idx = gl_GlobalInvocationID.x * 4;
|
| 20 |
+
|
| 21 |
+
if (idx >= p.ne) {
|
| 22 |
+
return;
|
| 23 |
+
}
|
| 24 |
+
|
| 25 |
+
// Check if all four components are in bounds and aligned,
|
| 26 |
+
// then use vector loads
|
| 27 |
+
if (idx + 3 < p.ne && (p.ne % 4) == 0) {
|
| 28 |
+
vec4 result = vec4(0.0f);
|
| 29 |
+
|
| 30 |
+
[[unroll]] for (uint i = 0; i < p.k_num; i++) {
|
| 31 |
+
result += data_a4[(i * p.ne + idx) / 4];
|
| 32 |
+
}
|
| 33 |
+
|
| 34 |
+
data_d4[idx / 4] = result;
|
| 35 |
+
} else {
|
| 36 |
+
[[unroll]] for (uint j = 0; j < 4; ++j) {
|
| 37 |
+
if (idx + j < p.ne) {
|
| 38 |
+
float result = 0.0f;
|
| 39 |
+
|
| 40 |
+
[[unroll]] for (uint i = 0; i < p.k_num; i++) {
|
| 41 |
+
result += data_a[i * p.ne + idx + j];
|
| 42 |
+
}
|
| 43 |
+
|
| 44 |
+
data_d[idx + j] = result;
|
| 45 |
+
}
|
| 46 |
+
}
|
| 47 |
+
}
|
| 48 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec.comp
ADDED
|
@@ -0,0 +1,264 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
|
| 3 |
+
#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
|
| 4 |
+
|
| 5 |
+
#include "mul_mat_vec_base.glsl"
|
| 6 |
+
#include "dequant_funcs.glsl"
|
| 7 |
+
|
| 8 |
+
layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
|
| 9 |
+
|
| 10 |
+
#if !defined(DATA_A_F32) && !defined(DATA_A_F16) && !defined(DATA_A_BF16)
|
| 11 |
+
#define K_PER_ITER 8
|
| 12 |
+
#else
|
| 13 |
+
#define K_PER_ITER 4
|
| 14 |
+
#endif
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
uint a_offset, b_offset, d_offset, y_offset;
|
| 18 |
+
|
| 19 |
+
vec4 load_b(const uint j, const uint iybs, const uint iqs, const bool lastiter, out bool OOB_y, out bool OOB_z, out bool OOB_w) {
|
| 20 |
+
// Check if the latter elements are OOB, and don't fetch B or accumulate it.
|
| 21 |
+
OOB_y = lastiter && (iybs + iqs + y_offset >= p.ncols);
|
| 22 |
+
OOB_z = lastiter && (iybs + iqs + y_offset*2 >= p.ncols);
|
| 23 |
+
OOB_w = lastiter && (iybs + iqs + y_offset*3 >= p.ncols);
|
| 24 |
+
|
| 25 |
+
if (!OOB_w) {
|
| 26 |
+
return vec4(FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs]),
|
| 27 |
+
FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs + y_offset]),
|
| 28 |
+
FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs + y_offset*2]),
|
| 29 |
+
FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs + y_offset*3]));
|
| 30 |
+
} else if (!OOB_z) {
|
| 31 |
+
return vec4(FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs]),
|
| 32 |
+
FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs + y_offset]),
|
| 33 |
+
FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs + y_offset*2]),
|
| 34 |
+
0);
|
| 35 |
+
} else if (!OOB_y) {
|
| 36 |
+
return vec4(FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs]),
|
| 37 |
+
FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs + y_offset]),
|
| 38 |
+
0, 0);
|
| 39 |
+
} else {
|
| 40 |
+
return vec4(FLOAT_TYPE(data_b[j*p.batch_stride_b + b_offset + iybs + iqs]),
|
| 41 |
+
0, 0, 0);
|
| 42 |
+
}
|
| 43 |
+
}
|
| 44 |
+
|
| 45 |
+
void iter(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const uint first_row, const uint num_rows, const uint tid, const uint i, bool lastiter)
|
| 46 |
+
{
|
| 47 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 48 |
+
const uint col = i*BLOCK_SIZE + K_PER_ITER*tid;
|
| 49 |
+
const uint iqs = (col%QUANT_K)/QUANT_R; // quant index
|
| 50 |
+
const uint iybs = col - col%QUANT_K; // y block start index
|
| 51 |
+
|
| 52 |
+
#if K_PER_ITER == 8
|
| 53 |
+
#if QUANT_R == 2
|
| 54 |
+
// Note that we end up fetching bogus elements here, but its fine as they'll be
|
| 55 |
+
// within an accessible block.
|
| 56 |
+
const vec4 bv02 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + iybs + iqs) / 4]);
|
| 57 |
+
const vec4 bv13 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + iybs + iqs + y_offset) / 4]);
|
| 58 |
+
const vec4 bv0 = vec4(bv02.x, bv13.x, bv02.y, bv13.y);
|
| 59 |
+
const vec4 bv1 = vec4(bv02.z, bv13.z, bv02.w, bv13.w);
|
| 60 |
+
#else
|
| 61 |
+
const vec4 bv0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + iybs + iqs) / 4]);
|
| 62 |
+
const vec4 bv1 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + iybs + iqs) / 4 + 1]);
|
| 63 |
+
#endif
|
| 64 |
+
#else
|
| 65 |
+
bool OOB_y;
|
| 66 |
+
bool OOB_z;
|
| 67 |
+
bool OOB_w;
|
| 68 |
+
|
| 69 |
+
const vec4 b = load_b(j, iybs, iqs, lastiter, OOB_y, OOB_z, OOB_w);
|
| 70 |
+
#endif
|
| 71 |
+
uint ibi = first_row*p.ncols;
|
| 72 |
+
[[unroll]] for (uint n = 0; n < num_rows; ++n) {
|
| 73 |
+
const uint ib = (ibi + col)/QUANT_K; // block index
|
| 74 |
+
ibi += p.ncols;
|
| 75 |
+
|
| 76 |
+
#if K_PER_ITER == 8
|
| 77 |
+
vec4 v = dequantize4(ib, iqs, a_offset);
|
| 78 |
+
vec4 v2 = dequantize4(ib, iqs+(4/QUANT_R), a_offset);
|
| 79 |
+
|
| 80 |
+
const vec2 dm = get_dm(ib, a_offset);
|
| 81 |
+
if (dm.y != 0) { // quant has min component
|
| 82 |
+
v = v * dm.x + dm.y;
|
| 83 |
+
v2 = v2 * dm.x + dm.y;
|
| 84 |
+
}
|
| 85 |
+
|
| 86 |
+
// matrix multiplication
|
| 87 |
+
FLOAT_TYPE rowtmp = dot(bv0, v);
|
| 88 |
+
rowtmp += dot(bv1, v2);
|
| 89 |
+
|
| 90 |
+
if (dm.y == 0)
|
| 91 |
+
rowtmp *= dm.x;
|
| 92 |
+
|
| 93 |
+
temp[j][n] += rowtmp;
|
| 94 |
+
#else
|
| 95 |
+
if (!OOB_w) {
|
| 96 |
+
const vec4 v = dequantize4(ib, iqs, a_offset);
|
| 97 |
+
temp[j][n] += dot(v, b);
|
| 98 |
+
} else if (!OOB_z) {
|
| 99 |
+
const vec2 v0 = dequantize(ib, iqs, a_offset);
|
| 100 |
+
const FLOAT_TYPE v1 = dequantize1(ib + 2/QUANT_R, iqs, a_offset);
|
| 101 |
+
const vec3 v = vec3(v0.x, v0.y, v1);
|
| 102 |
+
const vec3 b0 = vec3(b.x, b.y, b.z);
|
| 103 |
+
temp[j][n] += dot(v, b0);
|
| 104 |
+
} else if (!OOB_y) {
|
| 105 |
+
const vec2 v0 = dequantize(ib, iqs, a_offset);
|
| 106 |
+
const vec2 b0 = vec2(b.x, b.y);
|
| 107 |
+
temp[j][n] += dot(v0, b0);
|
| 108 |
+
} else {
|
| 109 |
+
const FLOAT_TYPE v = dequantize1(ib, iqs, a_offset);
|
| 110 |
+
temp[j][n] = fma(v, b.x, temp[j][n]);
|
| 111 |
+
}
|
| 112 |
+
#endif
|
| 113 |
+
}
|
| 114 |
+
}
|
| 115 |
+
}
|
| 116 |
+
|
| 117 |
+
#if defined(DATA_A_F32) || defined(DATA_A_F16) || defined(DATA_A_BF16)
|
| 118 |
+
void iter_aligned_nonquant(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const uint first_row, const uint num_rows, const uint tid, const uint i)
|
| 119 |
+
{
|
| 120 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 121 |
+
const uint col = i*BLOCK_SIZE + K_PER_ITER*tid;
|
| 122 |
+
const uint iqs = 0; // quant index
|
| 123 |
+
const uint iybs = col; // y block start index
|
| 124 |
+
|
| 125 |
+
const vec4 b = data_b_v4[(j*p.batch_stride_b + b_offset + iybs + iqs) / 4];
|
| 126 |
+
|
| 127 |
+
uint ibi = first_row*p.ncols;
|
| 128 |
+
[[unroll]] for (uint n = 0; n < num_rows; ++n) {
|
| 129 |
+
const uint ib = (ibi + col)/QUANT_K; // block index
|
| 130 |
+
ibi += p.ncols;
|
| 131 |
+
|
| 132 |
+
const vec4 v = dequantize4_2aligned(ib, iqs, a_offset);
|
| 133 |
+
|
| 134 |
+
// matrix multiplication
|
| 135 |
+
temp[j][n] += dot(v, b);
|
| 136 |
+
}
|
| 137 |
+
}
|
| 138 |
+
}
|
| 139 |
+
#endif
|
| 140 |
+
|
| 141 |
+
void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
|
| 142 |
+
const uint tid = gl_LocalInvocationID.x;
|
| 143 |
+
|
| 144 |
+
get_offsets(a_offset, b_offset, d_offset);
|
| 145 |
+
const bool is_aligned_nonquant =
|
| 146 |
+
p.batch_stride_b % 4 == 0 && b_offset % 4 == 0 &&
|
| 147 |
+
p.ncols % 4 == 0 && BLOCK_SIZE % 4 == 0 &&
|
| 148 |
+
K_PER_ITER == 4;
|
| 149 |
+
|
| 150 |
+
y_offset = QUANT_R == 1 ? 1 : QUANT_K/2;
|
| 151 |
+
|
| 152 |
+
FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];
|
| 153 |
+
|
| 154 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 155 |
+
[[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) {
|
| 156 |
+
temp[j][i] = FLOAT_TYPE(0);
|
| 157 |
+
}
|
| 158 |
+
}
|
| 159 |
+
|
| 160 |
+
uint num_iters = p.ncols / (K_PER_ITER * BLOCK_SIZE);
|
| 161 |
+
if (num_iters * K_PER_ITER * BLOCK_SIZE + K_PER_ITER*tid < p.ncols) {
|
| 162 |
+
num_iters++;
|
| 163 |
+
}
|
| 164 |
+
int unroll_count = 4;
|
| 165 |
+
uint unrolled_iters = num_iters & ~(unroll_count - 1);
|
| 166 |
+
|
| 167 |
+
uint i = 0;
|
| 168 |
+
|
| 169 |
+
#if K_PER_ITER == 4
|
| 170 |
+
// If the K dimension is odd, we need lastiter==true on the last iteration
|
| 171 |
+
// so OOB is computed correctly. Skip some unrolling to make that happen.
|
| 172 |
+
if ((p.ncols & 3) != 0 &&
|
| 173 |
+
unrolled_iters == num_iters &&
|
| 174 |
+
unrolled_iters > 0) {
|
| 175 |
+
unrolled_iters -= unroll_count;
|
| 176 |
+
}
|
| 177 |
+
if (is_aligned_nonquant) {
|
| 178 |
+
while (i < unrolled_iters) {
|
| 179 |
+
// Manually partially unroll the loop
|
| 180 |
+
[[unroll]] for (uint k = 0; k < unroll_count; ++k) {
|
| 181 |
+
iter_aligned_nonquant(temp, first_row, num_rows, tid, i*K_PER_ITER);
|
| 182 |
+
i++;
|
| 183 |
+
}
|
| 184 |
+
}
|
| 185 |
+
} else {
|
| 186 |
+
#endif
|
| 187 |
+
while (i < unrolled_iters) {
|
| 188 |
+
// Manually partially unroll the loop
|
| 189 |
+
[[unroll]] for (uint k = 0; k < unroll_count; ++k) {
|
| 190 |
+
iter(temp, first_row, num_rows, tid, i*K_PER_ITER, false);
|
| 191 |
+
i++;
|
| 192 |
+
}
|
| 193 |
+
}
|
| 194 |
+
#if K_PER_ITER == 4
|
| 195 |
+
}
|
| 196 |
+
#endif
|
| 197 |
+
|
| 198 |
+
unroll_count = 2;
|
| 199 |
+
unrolled_iters = num_iters & ~(unroll_count - 1);
|
| 200 |
+
|
| 201 |
+
#if K_PER_ITER == 4
|
| 202 |
+
if ((p.ncols & 3) != 0 &&
|
| 203 |
+
unrolled_iters == num_iters &&
|
| 204 |
+
unrolled_iters > 0) {
|
| 205 |
+
unrolled_iters -= unroll_count;
|
| 206 |
+
}
|
| 207 |
+
|
| 208 |
+
if (is_aligned_nonquant) {
|
| 209 |
+
while (i < unrolled_iters && is_aligned_nonquant) {
|
| 210 |
+
// Manually partially unroll the loop
|
| 211 |
+
[[unroll]] for (uint k = 0; k < unroll_count; ++k) {
|
| 212 |
+
iter_aligned_nonquant(temp, first_row, num_rows, tid, i*K_PER_ITER);
|
| 213 |
+
i++;
|
| 214 |
+
}
|
| 215 |
+
}
|
| 216 |
+
} else {
|
| 217 |
+
#endif
|
| 218 |
+
while (i < unrolled_iters) {
|
| 219 |
+
// Manually partially unroll the loop
|
| 220 |
+
[[unroll]] for (uint k = 0; k < unroll_count; ++k) {
|
| 221 |
+
iter(temp, first_row, num_rows, tid, i*K_PER_ITER, false);
|
| 222 |
+
i++;
|
| 223 |
+
}
|
| 224 |
+
}
|
| 225 |
+
#if K_PER_ITER == 4
|
| 226 |
+
}
|
| 227 |
+
#endif
|
| 228 |
+
|
| 229 |
+
#if K_PER_ITER == 4
|
| 230 |
+
if (is_aligned_nonquant) {
|
| 231 |
+
while (i < num_iters) {
|
| 232 |
+
iter_aligned_nonquant(temp, first_row, num_rows, tid, i*K_PER_ITER);
|
| 233 |
+
i++;
|
| 234 |
+
}
|
| 235 |
+
} else {
|
| 236 |
+
#endif
|
| 237 |
+
while (i < num_iters) {
|
| 238 |
+
iter(temp, first_row, num_rows, tid, i*K_PER_ITER, true);
|
| 239 |
+
i++;
|
| 240 |
+
}
|
| 241 |
+
#if K_PER_ITER == 4
|
| 242 |
+
}
|
| 243 |
+
#endif
|
| 244 |
+
|
| 245 |
+
reduce_result(temp, d_offset, first_row, num_rows, tid);
|
| 246 |
+
}
|
| 247 |
+
|
| 248 |
+
void main() {
|
| 249 |
+
const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z);
|
| 250 |
+
|
| 251 |
+
#ifdef NEEDS_INIT_IQ_SHMEM
|
| 252 |
+
init_iq_shmem(gl_WorkGroupSize);
|
| 253 |
+
#endif
|
| 254 |
+
|
| 255 |
+
// do NUM_ROWS at a time, unless there aren't enough remaining rows
|
| 256 |
+
if (first_row + NUM_ROWS <= p.stride_d) {
|
| 257 |
+
compute_outputs(first_row, NUM_ROWS);
|
| 258 |
+
} else {
|
| 259 |
+
if (first_row >= p.stride_d) {
|
| 260 |
+
return;
|
| 261 |
+
}
|
| 262 |
+
compute_outputs(first_row, min(NUM_ROWS, p.stride_d - first_row));
|
| 263 |
+
}
|
| 264 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_base.glsl
ADDED
|
@@ -0,0 +1,230 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#extension GL_EXT_control_flow_attributes : enable
|
| 2 |
+
#extension GL_EXT_shader_16bit_storage : require
|
| 3 |
+
#extension GL_EXT_shader_8bit_storage : require
|
| 4 |
+
|
| 5 |
+
#if USE_SUBGROUP_ADD || USE_SUBGROUP_ADD_NO_SHMEM
|
| 6 |
+
#extension GL_KHR_shader_subgroup_basic : require
|
| 7 |
+
#extension GL_KHR_shader_subgroup_arithmetic : require
|
| 8 |
+
#endif
|
| 9 |
+
|
| 10 |
+
#ifdef MUL_MAT_ID
|
| 11 |
+
#define EXPERT_COUNT 8
|
| 12 |
+
#endif
|
| 13 |
+
|
| 14 |
+
#include "mul_mat_vec_iface.glsl"
|
| 15 |
+
|
| 16 |
+
layout (push_constant) uniform parameter
|
| 17 |
+
{
|
| 18 |
+
uint ncols;
|
| 19 |
+
uint stride_a;
|
| 20 |
+
uint stride_b;
|
| 21 |
+
uint stride_d;
|
| 22 |
+
|
| 23 |
+
uint batch_stride_a;
|
| 24 |
+
uint batch_stride_b;
|
| 25 |
+
uint batch_stride_d;
|
| 26 |
+
|
| 27 |
+
uint fusion_flags;
|
| 28 |
+
|
| 29 |
+
#ifdef MUL_MAT_ID
|
| 30 |
+
uint nei0;
|
| 31 |
+
uint ne11;
|
| 32 |
+
uint expert_i1;
|
| 33 |
+
uint nbi1;
|
| 34 |
+
#else
|
| 35 |
+
uint base_work_group_y;
|
| 36 |
+
uint ne02;
|
| 37 |
+
uint ne12;
|
| 38 |
+
uint broadcast2;
|
| 39 |
+
uint broadcast3;
|
| 40 |
+
#endif
|
| 41 |
+
} p;
|
| 42 |
+
|
| 43 |
+
#ifdef MUL_MAT_ID
|
| 44 |
+
uint expert_id;
|
| 45 |
+
#endif
|
| 46 |
+
|
| 47 |
+
void get_offsets(out uint a_offset, out uint b_offset, out uint d_offset) {
|
| 48 |
+
#ifdef MUL_MAT_ID
|
| 49 |
+
const uint expert_i0 = gl_WorkGroupID.y;
|
| 50 |
+
#else
|
| 51 |
+
const uint batch_idx = gl_WorkGroupID.y + p.base_work_group_y;
|
| 52 |
+
#endif
|
| 53 |
+
|
| 54 |
+
#ifndef MUL_MAT_ID
|
| 55 |
+
uint batch_idx_a = 0;
|
| 56 |
+
if (batch_idx != 0) {
|
| 57 |
+
const uint i13 = batch_idx / p.ne12;
|
| 58 |
+
const uint i12 = batch_idx % p.ne12;
|
| 59 |
+
|
| 60 |
+
const uint i03 = i13 / p.broadcast3;
|
| 61 |
+
const uint i02 = i12 / p.broadcast2;
|
| 62 |
+
|
| 63 |
+
batch_idx_a = i03 * p.ne02 + i02;
|
| 64 |
+
}
|
| 65 |
+
#else
|
| 66 |
+
expert_id = data_ids[expert_i0 + p.expert_i1 * p.nbi1];
|
| 67 |
+
#endif
|
| 68 |
+
|
| 69 |
+
a_offset =
|
| 70 |
+
#ifdef MUL_MAT_ID
|
| 71 |
+
expert_id * (p.batch_stride_a / QUANT_K);
|
| 72 |
+
#else
|
| 73 |
+
batch_idx_a * (p.batch_stride_a / QUANT_K);
|
| 74 |
+
#endif
|
| 75 |
+
b_offset =
|
| 76 |
+
#ifdef MUL_MAT_ID
|
| 77 |
+
(expert_i0 % p.ne11) * p.stride_b + p.expert_i1 * p.batch_stride_b;
|
| 78 |
+
#else
|
| 79 |
+
batch_idx * p.batch_stride_b;
|
| 80 |
+
#endif
|
| 81 |
+
d_offset =
|
| 82 |
+
#ifdef MUL_MAT_ID
|
| 83 |
+
expert_i0 * p.stride_d + p.expert_i1 * p.batch_stride_d;
|
| 84 |
+
#else
|
| 85 |
+
batch_idx * p.batch_stride_d;
|
| 86 |
+
#endif
|
| 87 |
+
}
|
| 88 |
+
|
| 89 |
+
layout (constant_id = 0) const uint BLOCK_SIZE = 32;
|
| 90 |
+
layout (constant_id = 1) const uint NUM_ROWS = 1;
|
| 91 |
+
layout (constant_id = 2) const uint NUM_COLS = 1;
|
| 92 |
+
|
| 93 |
+
#ifdef USE_SUBGROUP_ADD_NO_SHMEM
|
| 94 |
+
void reduce_result(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const in uint32_t d_offset, const in uint32_t first_row, const in uint32_t num_rows, const in uint32_t tid) {
|
| 95 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 96 |
+
[[unroll]] for (uint n = 0; n < num_rows; ++n) {
|
| 97 |
+
temp[j][n] = subgroupAdd(temp[j][n]);
|
| 98 |
+
}
|
| 99 |
+
}
|
| 100 |
+
|
| 101 |
+
if (tid == 0) {
|
| 102 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 103 |
+
[[unroll]] for (uint n = 0; n < num_rows; ++n) {
|
| 104 |
+
#ifdef MUL_MAT_ID
|
| 105 |
+
if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0) {
|
| 106 |
+
temp[j][n] += FLOAT_TYPE(data_fuse0[expert_id*p.stride_d + first_row + n]);
|
| 107 |
+
}
|
| 108 |
+
if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE0) != 0) {
|
| 109 |
+
const uint expert_i0 = gl_GlobalInvocationID.y;
|
| 110 |
+
temp[j][n] *= FLOAT_TYPE(data_fuse0[expert_i0]);
|
| 111 |
+
}
|
| 112 |
+
if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE1) != 0) {
|
| 113 |
+
const uint expert_i0 = gl_GlobalInvocationID.y;
|
| 114 |
+
temp[j][n] *= FLOAT_TYPE(data_fuse1[expert_i0]);
|
| 115 |
+
}
|
| 116 |
+
#else
|
| 117 |
+
if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0) {
|
| 118 |
+
temp[j][n] += FLOAT_TYPE(data_fuse0[j*p.batch_stride_d + d_offset + first_row + n]);
|
| 119 |
+
}
|
| 120 |
+
if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS1) != 0) {
|
| 121 |
+
temp[j][n] += FLOAT_TYPE(data_fuse1[j*p.batch_stride_d + d_offset + first_row + n]);
|
| 122 |
+
}
|
| 123 |
+
#endif
|
| 124 |
+
data_d[j*p.batch_stride_d + d_offset + first_row + n] = D_TYPE(temp[j][n]);
|
| 125 |
+
}
|
| 126 |
+
}
|
| 127 |
+
}
|
| 128 |
+
}
|
| 129 |
+
#else
|
| 130 |
+
shared FLOAT_TYPE tmpsh[NUM_COLS][NUM_ROWS][BLOCK_SIZE];
|
| 131 |
+
|
| 132 |
+
void reduce_result(FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const in uint32_t d_offset, const in uint32_t first_row, const in uint32_t num_rows, const in uint32_t tid) {
|
| 133 |
+
// subgroupAdd is probably faster on devices that support it,
|
| 134 |
+
// particularly when the workgroup has more than one subgroup
|
| 135 |
+
#if USE_SUBGROUP_ADD
|
| 136 |
+
// sum up partial sums within a subgroup
|
| 137 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 138 |
+
[[unroll]] for (uint n = 0; n < num_rows; ++n) {
|
| 139 |
+
temp[j][n] = subgroupAdd(temp[j][n]);
|
| 140 |
+
}
|
| 141 |
+
}
|
| 142 |
+
|
| 143 |
+
// Go through shared memory to sum partials across subgroups
|
| 144 |
+
if (gl_SubgroupInvocationID == 0) {
|
| 145 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 146 |
+
[[unroll]] for (uint n = 0; n < num_rows; ++n) {
|
| 147 |
+
tmpsh[j][n][gl_SubgroupID] = temp[j][n];
|
| 148 |
+
}
|
| 149 |
+
}
|
| 150 |
+
}
|
| 151 |
+
barrier();
|
| 152 |
+
if (tid == 0) {
|
| 153 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 154 |
+
[[unroll]] for (uint n = 0; n < num_rows; ++n) {
|
| 155 |
+
temp[j][n] = FLOAT_TYPE(0);
|
| 156 |
+
[[unroll]] for (uint s = 0; s < gl_NumSubgroups; ++s) {
|
| 157 |
+
temp[j][n] += tmpsh[j][n][s];
|
| 158 |
+
}
|
| 159 |
+
#ifdef MUL_MAT_ID
|
| 160 |
+
if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0) {
|
| 161 |
+
temp[j][n] += FLOAT_TYPE(data_fuse0[expert_id*p.stride_d + first_row + n]);
|
| 162 |
+
}
|
| 163 |
+
if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE0) != 0) {
|
| 164 |
+
const uint expert_i0 = gl_GlobalInvocationID.y;
|
| 165 |
+
temp[j][n] *= FLOAT_TYPE(data_fuse0[expert_i0]);
|
| 166 |
+
}
|
| 167 |
+
if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE1) != 0) {
|
| 168 |
+
const uint expert_i0 = gl_GlobalInvocationID.y;
|
| 169 |
+
temp[j][n] *= FLOAT_TYPE(data_fuse1[expert_i0]);
|
| 170 |
+
}
|
| 171 |
+
#else
|
| 172 |
+
if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0) {
|
| 173 |
+
temp[j][n] += FLOAT_TYPE(data_fuse0[j*p.batch_stride_d + d_offset + first_row + n]);
|
| 174 |
+
}
|
| 175 |
+
if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS1) != 0) {
|
| 176 |
+
temp[j][n] += FLOAT_TYPE(data_fuse1[j*p.batch_stride_d + d_offset + first_row + n]);
|
| 177 |
+
}
|
| 178 |
+
#endif
|
| 179 |
+
data_d[j*p.batch_stride_d + d_offset + first_row + n] = D_TYPE(temp[j][n]);
|
| 180 |
+
}
|
| 181 |
+
}
|
| 182 |
+
}
|
| 183 |
+
#else
|
| 184 |
+
// sum up partial sums and write back result
|
| 185 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 186 |
+
[[unroll]] for (uint n = 0; n < num_rows; ++n) {
|
| 187 |
+
tmpsh[j][n][tid] = temp[j][n];
|
| 188 |
+
}
|
| 189 |
+
}
|
| 190 |
+
barrier();
|
| 191 |
+
[[unroll]] for (uint s = BLOCK_SIZE/2; s > 0; s >>= 1) {
|
| 192 |
+
if (tid < s) {
|
| 193 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 194 |
+
[[unroll]] for (uint n = 0; n < num_rows; ++n) {
|
| 195 |
+
tmpsh[j][n][tid] += tmpsh[j][n][tid + s];
|
| 196 |
+
}
|
| 197 |
+
}
|
| 198 |
+
}
|
| 199 |
+
barrier();
|
| 200 |
+
}
|
| 201 |
+
if (tid == 0) {
|
| 202 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 203 |
+
[[unroll]] for (uint n = 0; n < num_rows; ++n) {
|
| 204 |
+
#ifdef MUL_MAT_ID
|
| 205 |
+
if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0) {
|
| 206 |
+
tmpsh[j][n][0] += FLOAT_TYPE(data_fuse0[expert_id*p.stride_d + first_row + n]);
|
| 207 |
+
}
|
| 208 |
+
if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE0) != 0) {
|
| 209 |
+
const uint expert_i0 = gl_GlobalInvocationID.y;
|
| 210 |
+
tmpsh[j][n][0] *= FLOAT_TYPE(data_fuse0[expert_i0]);
|
| 211 |
+
}
|
| 212 |
+
if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_SCALE1) != 0) {
|
| 213 |
+
const uint expert_i0 = gl_GlobalInvocationID.y;
|
| 214 |
+
tmpsh[j][n][0] *= FLOAT_TYPE(data_fuse1[expert_i0]);
|
| 215 |
+
}
|
| 216 |
+
#else
|
| 217 |
+
if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0) {
|
| 218 |
+
tmpsh[j][n][0] += FLOAT_TYPE(data_fuse0[j*p.batch_stride_d + d_offset + first_row + n]);
|
| 219 |
+
}
|
| 220 |
+
if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS1) != 0) {
|
| 221 |
+
tmpsh[j][n][0] += FLOAT_TYPE(data_fuse1[j*p.batch_stride_d + d_offset + first_row + n]);
|
| 222 |
+
}
|
| 223 |
+
#endif
|
| 224 |
+
data_d[j*p.batch_stride_d + d_offset + first_row + n] = D_TYPE(tmpsh[j][n][0]);
|
| 225 |
+
}
|
| 226 |
+
}
|
| 227 |
+
}
|
| 228 |
+
#endif
|
| 229 |
+
}
|
| 230 |
+
#endif
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iface.glsl
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#include "types.glsl"
|
| 2 |
+
|
| 3 |
+
#define MAT_VEC_FUSION_FLAGS_BIAS0 0x1
|
| 4 |
+
#define MAT_VEC_FUSION_FLAGS_BIAS1 0x2
|
| 5 |
+
#define MAT_VEC_FUSION_FLAGS_SCALE0 0x4
|
| 6 |
+
#define MAT_VEC_FUSION_FLAGS_SCALE1 0x8
|
| 7 |
+
|
| 8 |
+
layout (binding = 0) readonly buffer A {A_TYPE data_a[];};
|
| 9 |
+
#if defined(A_TYPEV4)
|
| 10 |
+
layout (binding = 0) readonly buffer AV4 {A_TYPEV4 data_a_v4[];};
|
| 11 |
+
#endif
|
| 12 |
+
#if defined(A_TYPE_PACKED16)
|
| 13 |
+
layout (binding = 0) readonly buffer A_PACKED16 {A_TYPE_PACKED16 data_a_packed16[];};
|
| 14 |
+
#endif
|
| 15 |
+
#if defined(A_TYPE_PACKED32)
|
| 16 |
+
layout (binding = 0) readonly buffer A_PACKED32 {A_TYPE_PACKED32 data_a_packed32[];};
|
| 17 |
+
#endif
|
| 18 |
+
|
| 19 |
+
layout (binding = 1) readonly buffer B {B_TYPE data_b[];};
|
| 20 |
+
#ifdef B_TYPEV2
|
| 21 |
+
layout (binding = 1) readonly buffer BV2 {B_TYPEV2 data_b_v2[];};
|
| 22 |
+
#endif
|
| 23 |
+
#ifdef B_TYPEV4
|
| 24 |
+
layout (binding = 1) readonly buffer BV4 {B_TYPEV4 data_b_v4[];};
|
| 25 |
+
#endif
|
| 26 |
+
|
| 27 |
+
layout (binding = 2) writeonly buffer D {D_TYPE data_d[];};
|
| 28 |
+
|
| 29 |
+
layout (binding = 3) readonly buffer Fuse0 {D_TYPE data_fuse0[];};
|
| 30 |
+
layout (binding = 4) readonly buffer Fuse1 {D_TYPE data_fuse1[];};
|
| 31 |
+
|
| 32 |
+
#ifdef MUL_MAT_ID
|
| 33 |
+
layout (binding = 5) readonly buffer IDS {int data_ids[];};
|
| 34 |
+
#endif
|
| 35 |
+
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_m.comp
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
|
| 3 |
+
|
| 4 |
+
#include "mul_mat_vec_base.glsl"
|
| 5 |
+
|
| 6 |
+
layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
|
| 7 |
+
|
| 8 |
+
FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];
|
| 9 |
+
|
| 10 |
+
void calc_superblock(const uint a_offset, const uint b_offset, const uint ib32, const uint i,
|
| 11 |
+
const uint num_blocks_per_row, const uint first_row, const uint num_rows) {
|
| 12 |
+
// Compute starting index in matrix B for this superblock
|
| 13 |
+
const uint y_idx = i * QUANT_K + 32 * ib32;
|
| 14 |
+
uint ibi = a_offset + first_row * num_blocks_per_row + i;
|
| 15 |
+
|
| 16 |
+
// Precompute indices for quantization lookup tables
|
| 17 |
+
const uint qh_base = 2 * ib32;
|
| 18 |
+
const uint qs_base = 4 * ib32;
|
| 19 |
+
const uint sc_index = ib32 / 2;
|
| 20 |
+
const uint sc_shift = 6 * (ib32 & 1);
|
| 21 |
+
|
| 22 |
+
// Loop over rows in the superblock
|
| 23 |
+
[[unroll]] for (uint n = 0; n < num_rows; ++n) {
|
| 24 |
+
// Load per-block scales and shift for quantization
|
| 25 |
+
const uint16_t[4] scales = data_a[ibi].scales;
|
| 26 |
+
const u16vec4 s = u16vec4(scales[0], scales[1], scales[2], scales[3]) >> 12;
|
| 27 |
+
const float d = float(unpackHalf2x16(s.x | (s.y << 4) | (s.z << 8) | (s.w << 12)).x);
|
| 28 |
+
const uint sc = data_a[ibi].scales[sc_index] >> sc_shift;
|
| 29 |
+
|
| 30 |
+
// Temporary caches for decoding
|
| 31 |
+
FLOAT_TYPE dl_cache[4];
|
| 32 |
+
uint16_t gvf_cache[4];
|
| 33 |
+
float delta_cache[4];
|
| 34 |
+
|
| 35 |
+
// Precompute the multiplier and lookup values for 4 sub-blocks
|
| 36 |
+
[[unroll]] for (uint l = 0; l < 4; ++l) {
|
| 37 |
+
dl_cache[l] = FLOAT_TYPE(d * (2 * bitfieldExtract(sc, 3 * int(l / 2), 3) + 1));
|
| 38 |
+
const uint qh = data_a[ibi].qh[qh_base + l / 2] >> (4 * (l & 1));
|
| 39 |
+
const uint qs = data_a[ibi].qs[qs_base + l];
|
| 40 |
+
gvf_cache[l] = iq1s_grid[qs | ((qh & 7) << 8)];
|
| 41 |
+
delta_cache[l] = ((qh & 8) != 0) ? -IQ1M_DELTA : IQ1M_DELTA;
|
| 42 |
+
}
|
| 43 |
+
|
| 44 |
+
// Loop over columns of the output
|
| 45 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 46 |
+
// Compute base index for matrix B
|
| 47 |
+
const uint base_b_idx = (j * p.batch_stride_b + b_offset + y_idx) / 4;
|
| 48 |
+
vec4 b_vals[8];
|
| 49 |
+
|
| 50 |
+
// Load 8 vec4 values from matrix B
|
| 51 |
+
[[unroll]] for (int idx = 0; idx < 8; ++idx) {
|
| 52 |
+
b_vals[idx] = vec4(data_b_v4[base_b_idx + idx]);
|
| 53 |
+
}
|
| 54 |
+
|
| 55 |
+
FLOAT_TYPE col_sum = FLOAT_TYPE(0.0);
|
| 56 |
+
|
| 57 |
+
// Loop over sub-blocks
|
| 58 |
+
[[unroll]] for (uint l = 0; l < 4; ++l) {
|
| 59 |
+
const uint16_t grid = gvf_cache[l];
|
| 60 |
+
const float dl = dl_cache[l];
|
| 61 |
+
|
| 62 |
+
// Decode 8 2-bit fbits from gvf_cache
|
| 63 |
+
float f0 = float(bitfieldExtract(grid, 0, 2));
|
| 64 |
+
float f1 = float(bitfieldExtract(grid, 2, 2));
|
| 65 |
+
float f2 = float(bitfieldExtract(grid, 4, 2));
|
| 66 |
+
float f3 = float(bitfieldExtract(grid, 6, 2));
|
| 67 |
+
float f4 = float(bitfieldExtract(grid, 8, 2));
|
| 68 |
+
float f5 = float(bitfieldExtract(grid, 10, 2));
|
| 69 |
+
float f6 = float(bitfieldExtract(grid, 12, 2));
|
| 70 |
+
float f7 = float(bitfieldExtract(grid, 14, 2));
|
| 71 |
+
|
| 72 |
+
// Pack into vec4 for vectorized FMA
|
| 73 |
+
const vec4 fbits_v0 = vec4(f0, f1, f2, f3);
|
| 74 |
+
const vec4 fbits_v1 = vec4(f4, f5, f6, f7);
|
| 75 |
+
const vec4 delta_v = vec4(delta_cache[l]);
|
| 76 |
+
|
| 77 |
+
// Vectorized fused multiply-add
|
| 78 |
+
vec4 sum_v = fma(b_vals[2*l + 0], fbits_v0 + delta_v, vec4(0.0));
|
| 79 |
+
sum_v = fma(b_vals[2*l + 1], fbits_v1 + delta_v, sum_v);
|
| 80 |
+
|
| 81 |
+
// Horizontal add to get scalar sum
|
| 82 |
+
FLOAT_TYPE sum = sum_v.x + sum_v.y + sum_v.z + sum_v.w;
|
| 83 |
+
|
| 84 |
+
// Accumulate to column sum
|
| 85 |
+
col_sum = fma(dl, sum, col_sum);
|
| 86 |
+
}
|
| 87 |
+
// Write result to temporary buffer
|
| 88 |
+
temp[j][n] += col_sum;
|
| 89 |
+
}
|
| 90 |
+
ibi += num_blocks_per_row;
|
| 91 |
+
}
|
| 92 |
+
}
|
| 93 |
+
|
| 94 |
+
void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
|
| 95 |
+
uint a_offset, b_offset, d_offset;
|
| 96 |
+
get_offsets(a_offset, b_offset, d_offset);
|
| 97 |
+
|
| 98 |
+
const uint num_blocks_per_row = p.ncols / QUANT_K;
|
| 99 |
+
|
| 100 |
+
// 8 threads are used to process each block
|
| 101 |
+
const uint blocks_per_wg = gl_WorkGroupSize.x/8;
|
| 102 |
+
const uint tid = gl_LocalInvocationID.x;
|
| 103 |
+
const uint itid = tid % 8; // 0...7
|
| 104 |
+
const uint ix = tid / 8;
|
| 105 |
+
|
| 106 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 107 |
+
[[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) {
|
| 108 |
+
temp[j][i] = FLOAT_TYPE(0);
|
| 109 |
+
}
|
| 110 |
+
}
|
| 111 |
+
|
| 112 |
+
[[unroll]] for (uint i = ix; i < num_blocks_per_row; i += blocks_per_wg)
|
| 113 |
+
calc_superblock(a_offset, b_offset, itid, i, num_blocks_per_row, first_row, num_rows);
|
| 114 |
+
|
| 115 |
+
reduce_result(temp, d_offset, first_row, num_rows, tid);
|
| 116 |
+
}
|
| 117 |
+
|
| 118 |
+
void main() {
|
| 119 |
+
const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z);
|
| 120 |
+
|
| 121 |
+
init_iq_shmem(gl_WorkGroupSize);
|
| 122 |
+
|
| 123 |
+
// do NUM_ROWS at a time, unless there aren't enough remaining rows
|
| 124 |
+
if (first_row + NUM_ROWS <= p.stride_d) {
|
| 125 |
+
compute_outputs(first_row, NUM_ROWS);
|
| 126 |
+
} else {
|
| 127 |
+
if (first_row >= p.stride_d) {
|
| 128 |
+
return;
|
| 129 |
+
}
|
| 130 |
+
compute_outputs(first_row, p.stride_d - first_row);
|
| 131 |
+
}
|
| 132 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq1_s.comp
ADDED
|
@@ -0,0 +1,95 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
|
| 3 |
+
|
| 4 |
+
#include "mul_mat_vec_base.glsl"
|
| 5 |
+
|
| 6 |
+
layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
|
| 7 |
+
|
| 8 |
+
FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];
|
| 9 |
+
|
| 10 |
+
void calc_superblock(const uint a_offset, const uint b_offset, const uint ib32, const uint i,
|
| 11 |
+
const uint num_blocks_per_row, const uint first_row, const uint num_rows) {
|
| 12 |
+
const uint y_idx_base = i * QUANT_K + 32 * ib32;
|
| 13 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 14 |
+
const uint base_b_idx = (j * p.batch_stride_b + b_offset + y_idx_base) / 4;
|
| 15 |
+
[[unroll]] for (uint l = 0; l < 4; ++l) {
|
| 16 |
+
const vec4 b_val_0 = vec4(data_b_v4[base_b_idx + 2 * l]);
|
| 17 |
+
const vec4 b_val_1 = vec4(data_b_v4[base_b_idx + 2 * l + 1]);
|
| 18 |
+
|
| 19 |
+
// index for data_a
|
| 20 |
+
uint ibi = a_offset + first_row * num_blocks_per_row + i;
|
| 21 |
+
|
| 22 |
+
[[unroll]] for (uint n = 0; n < num_rows; ++n) {
|
| 23 |
+
const float d = float(data_a[ibi].d);
|
| 24 |
+
const uint qh = data_a[ibi].qh[ib32];
|
| 25 |
+
|
| 26 |
+
const float dl = d * float(2 * bitfieldExtract(qh, 12, 3) + 1);
|
| 27 |
+
const uint qs = data_a[ibi].qs[4 * ib32 + l];
|
| 28 |
+
const uint idxhi = bitfieldExtract(qh, 3 * int(l), 3);
|
| 29 |
+
const uint16_t grid = uint16_t(iq1s_grid[qs | (idxhi << 8)]);
|
| 30 |
+
|
| 31 |
+
const float delta_val = ((qh & 0x8000) != 0) ? -IQ1S_DELTA : IQ1S_DELTA;
|
| 32 |
+
const vec4 delta_v = vec4(delta_val);
|
| 33 |
+
const vec4 fbits0 = vec4(
|
| 34 |
+
float(bitfieldExtract(grid, 0, 2)),
|
| 35 |
+
float(bitfieldExtract(grid, 2, 2)),
|
| 36 |
+
float(bitfieldExtract(grid, 4, 2)),
|
| 37 |
+
float(bitfieldExtract(grid, 6, 2))
|
| 38 |
+
);
|
| 39 |
+
const vec4 fbits1 = vec4(
|
| 40 |
+
float(bitfieldExtract(grid, 8, 2)),
|
| 41 |
+
float(bitfieldExtract(grid, 10, 2)),
|
| 42 |
+
float(bitfieldExtract(grid, 12, 2)),
|
| 43 |
+
float(bitfieldExtract(grid, 14, 2))
|
| 44 |
+
);
|
| 45 |
+
|
| 46 |
+
vec4 sum_v = fma(b_val_0, fbits0 + delta_v, vec4(0.0));
|
| 47 |
+
sum_v = fma(b_val_1, fbits1 + delta_v, sum_v);
|
| 48 |
+
FLOAT_TYPE sum = dot(sum_v, vec4(1.0));
|
| 49 |
+
|
| 50 |
+
temp[j][n] = fma(dl, sum, temp[j][n]);
|
| 51 |
+
ibi += num_blocks_per_row;
|
| 52 |
+
}
|
| 53 |
+
}
|
| 54 |
+
}
|
| 55 |
+
}
|
| 56 |
+
|
| 57 |
+
void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
|
| 58 |
+
uint a_offset, b_offset, d_offset;
|
| 59 |
+
get_offsets(a_offset, b_offset, d_offset);
|
| 60 |
+
|
| 61 |
+
const uint num_blocks_per_row = p.ncols / QUANT_K;
|
| 62 |
+
|
| 63 |
+
// 8 threads are used to process each block
|
| 64 |
+
const uint blocks_per_wg = gl_WorkGroupSize.x/8;
|
| 65 |
+
const uint tid = gl_LocalInvocationID.x;
|
| 66 |
+
const uint itid = tid % 8; // 0...7
|
| 67 |
+
const uint ix = tid / 8;
|
| 68 |
+
|
| 69 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 70 |
+
[[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) {
|
| 71 |
+
temp[j][i] = FLOAT_TYPE(0);
|
| 72 |
+
}
|
| 73 |
+
}
|
| 74 |
+
|
| 75 |
+
[[unroll]] for (uint i = ix; i < num_blocks_per_row; i += blocks_per_wg)
|
| 76 |
+
calc_superblock(a_offset, b_offset, itid, i, num_blocks_per_row, first_row, num_rows);
|
| 77 |
+
|
| 78 |
+
reduce_result(temp, d_offset, first_row, num_rows, tid);
|
| 79 |
+
}
|
| 80 |
+
|
| 81 |
+
void main() {
|
| 82 |
+
const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z);
|
| 83 |
+
|
| 84 |
+
init_iq_shmem(gl_WorkGroupSize);
|
| 85 |
+
|
| 86 |
+
// do NUM_ROWS at a time, unless there aren't enough remaining rows
|
| 87 |
+
if (first_row + NUM_ROWS <= p.stride_d) {
|
| 88 |
+
compute_outputs(first_row, NUM_ROWS);
|
| 89 |
+
} else {
|
| 90 |
+
if (first_row >= p.stride_d) {
|
| 91 |
+
return;
|
| 92 |
+
}
|
| 93 |
+
compute_outputs(first_row, p.stride_d - first_row);
|
| 94 |
+
}
|
| 95 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_s.comp
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
|
| 3 |
+
|
| 4 |
+
#include "mul_mat_vec_base.glsl"
|
| 5 |
+
|
| 6 |
+
layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
|
| 7 |
+
|
| 8 |
+
FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];
|
| 9 |
+
|
| 10 |
+
void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) {
|
| 11 |
+
const uint y_idx = i * QUANT_K + 16 * itid;
|
| 12 |
+
const uint nibble_shift = 4 * (itid & 1);
|
| 13 |
+
const uint ib32 = itid / 2; // 0..7
|
| 14 |
+
|
| 15 |
+
uint ibi = a_offset + first_row * num_blocks_per_row + i;
|
| 16 |
+
[[unroll]] for (uint n = 0; n < num_rows; ++n) {
|
| 17 |
+
const float d = float(data_a[ibi].d);
|
| 18 |
+
const uint scale = (data_a[ibi].scales[ib32] >> nibble_shift) & 0xF;
|
| 19 |
+
const float db = d * (0.5 + scale) * 0.25;
|
| 20 |
+
|
| 21 |
+
const uint qh = data_a[ibi].qh[ib32];
|
| 22 |
+
const u8vec2 qs16 = unpack8(uint32_t(data_a_packed16[ibi].qs[itid])).xy; // vec4 used due to #12147
|
| 23 |
+
const u8vec2 sign16 = unpack8(uint32_t(data_a_packed16[ibi].qs[QUANT_K / 16 + itid])).xy;
|
| 24 |
+
[[unroll]] for (uint l = 0; l < 2; ++l) {
|
| 25 |
+
const uint8_t sign = sign16[l];
|
| 26 |
+
const uint qs = qs16[l] | ((qh << (8 - nibble_shift - 2 * l)) & 0x300);
|
| 27 |
+
const uvec2 grid = iq2s_grid[qs];
|
| 28 |
+
const vec4 grid0 = vec4(unpack8(grid.x));
|
| 29 |
+
const vec4 grid1 = vec4(unpack8(grid.y));
|
| 30 |
+
|
| 31 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 32 |
+
vec4 b0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 0]);
|
| 33 |
+
vec4 b4 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 1]);
|
| 34 |
+
|
| 35 |
+
FLOAT_TYPE sum =
|
| 36 |
+
fma(FLOAT_TYPE(b0.x), FLOAT_TYPE((sign & 1) != 0 ? -grid0.x : grid0.x),
|
| 37 |
+
fma(FLOAT_TYPE(b0.y), FLOAT_TYPE((sign & 2) != 0 ? -grid0.y : grid0.y),
|
| 38 |
+
fma(FLOAT_TYPE(b0.z), FLOAT_TYPE((sign & 4) != 0 ? -grid0.z : grid0.z),
|
| 39 |
+
fma(FLOAT_TYPE(b0.w), FLOAT_TYPE((sign & 8) != 0 ? -grid0.w : grid0.w),
|
| 40 |
+
fma(FLOAT_TYPE(b4.x), FLOAT_TYPE((sign & 16) != 0 ? -grid1.x : grid1.x),
|
| 41 |
+
fma(FLOAT_TYPE(b4.y), FLOAT_TYPE((sign & 32) != 0 ? -grid1.y : grid1.y),
|
| 42 |
+
fma(FLOAT_TYPE(b4.z), FLOAT_TYPE((sign & 64) != 0 ? -grid1.z : grid1.z),
|
| 43 |
+
fma(FLOAT_TYPE(b4.w), FLOAT_TYPE((sign & 128) != 0 ? -grid1.w : grid1.w),
|
| 44 |
+
FLOAT_TYPE(0.0)))))))));
|
| 45 |
+
temp[j][n] = fma(db, sum, temp[j][n]);
|
| 46 |
+
}
|
| 47 |
+
}
|
| 48 |
+
ibi += num_blocks_per_row;
|
| 49 |
+
}
|
| 50 |
+
}
|
| 51 |
+
|
| 52 |
+
void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
|
| 53 |
+
uint a_offset, b_offset, d_offset;
|
| 54 |
+
get_offsets(a_offset, b_offset, d_offset);
|
| 55 |
+
|
| 56 |
+
const uint num_blocks_per_row = p.ncols / QUANT_K;
|
| 57 |
+
|
| 58 |
+
// 16 threads are used to process each block
|
| 59 |
+
const uint blocks_per_wg = gl_WorkGroupSize.x/16;
|
| 60 |
+
const uint tid = gl_LocalInvocationID.x;
|
| 61 |
+
const uint itid = tid % 16; // 0...15
|
| 62 |
+
const uint ix = tid / 16;
|
| 63 |
+
|
| 64 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 65 |
+
[[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) {
|
| 66 |
+
temp[j][i] = FLOAT_TYPE(0);
|
| 67 |
+
}
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
[[unroll]] for (uint i = ix; i < num_blocks_per_row; i += blocks_per_wg)
|
| 71 |
+
calc_superblock(a_offset, b_offset, itid, i, num_blocks_per_row, first_row, num_rows);
|
| 72 |
+
|
| 73 |
+
reduce_result(temp, d_offset, first_row, num_rows, tid);
|
| 74 |
+
}
|
| 75 |
+
|
| 76 |
+
void main() {
|
| 77 |
+
const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z);
|
| 78 |
+
|
| 79 |
+
init_iq_shmem(gl_WorkGroupSize);
|
| 80 |
+
|
| 81 |
+
// do NUM_ROWS at a time, unless there aren't enough remaining rows
|
| 82 |
+
if (first_row + NUM_ROWS <= p.stride_d) {
|
| 83 |
+
compute_outputs(first_row, NUM_ROWS);
|
| 84 |
+
} else {
|
| 85 |
+
if (first_row >= p.stride_d) {
|
| 86 |
+
return;
|
| 87 |
+
}
|
| 88 |
+
compute_outputs(first_row, p.stride_d - first_row);
|
| 89 |
+
}
|
| 90 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xs.comp
ADDED
|
@@ -0,0 +1,105 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
|
| 3 |
+
|
| 4 |
+
#include "mul_mat_vec_base.glsl"
|
| 5 |
+
|
| 6 |
+
layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
|
| 7 |
+
|
| 8 |
+
FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];
|
| 9 |
+
|
| 10 |
+
void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) {
|
| 11 |
+
const uint y_idx = i * QUANT_K + 16 * itid;
|
| 12 |
+
const uint nibble_shift = 4 * (itid & 1);
|
| 13 |
+
const uint ib32 = itid / 2; // 0..7
|
| 14 |
+
uint ibi = a_offset + first_row * num_blocks_per_row + i;
|
| 15 |
+
// Precompute db multiplication factors
|
| 16 |
+
float db_vals[NUM_ROWS];
|
| 17 |
+
[[unroll]] for (uint n = 0; n < num_rows; ++n) {
|
| 18 |
+
const float d = float(data_a[ibi].d);
|
| 19 |
+
const uint scale_raw = data_a[ibi].scales[ib32];
|
| 20 |
+
const uint scale = (scale_raw >> nibble_shift) & 0xF;
|
| 21 |
+
// Merge constant calculations d * (0.5 + scale) * 0.25 = d*0.125 + d*scale*0.25
|
| 22 |
+
db_vals[n] = d * (0.125f + float(scale) * 0.25f);
|
| 23 |
+
ibi += num_blocks_per_row;
|
| 24 |
+
}
|
| 25 |
+
ibi = a_offset + first_row * num_blocks_per_row + i;
|
| 26 |
+
[[unroll]] for (uint n = 0; n < num_rows; ++n) {
|
| 27 |
+
// Preload grid and sign data for all l values
|
| 28 |
+
vec4 grid0_vals[2], grid1_vals[2];
|
| 29 |
+
uint sign_vals[2], sign7_vals[2];
|
| 30 |
+
[[unroll]] for (uint l = 0; l < 2; ++l) {
|
| 31 |
+
const uint qs = data_a[ibi].qs[2 * itid + l];
|
| 32 |
+
sign_vals[l] = qs >> 9;
|
| 33 |
+
sign7_vals[l] = bitCount(sign_vals[l]);
|
| 34 |
+
const uvec2 grid_data = iq2xs_grid[qs & 511];
|
| 35 |
+
grid0_vals[l] = vec4(unpack8(grid_data.x));
|
| 36 |
+
grid1_vals[l] = vec4(unpack8(grid_data.y));
|
| 37 |
+
}
|
| 38 |
+
// Preload B data for all j columns (reduce repeated index calculations)
|
| 39 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 40 |
+
FLOAT_TYPE sum = FLOAT_TYPE(0.0);
|
| 41 |
+
[[unroll]] for (uint l = 0; l < 2; ++l) {
|
| 42 |
+
const uint sign = sign_vals[l];
|
| 43 |
+
const uint sign7 = sign7_vals[l];
|
| 44 |
+
const vec4 grid0 = grid0_vals[l];
|
| 45 |
+
const vec4 grid1 = grid1_vals[l];
|
| 46 |
+
// Precompute indices
|
| 47 |
+
const uint b_idx = (j * p.batch_stride_b + b_offset + y_idx) / 4 + 2 * l;
|
| 48 |
+
const vec4 b0 = vec4(data_b_v4[b_idx + 0]);
|
| 49 |
+
const vec4 b4 = vec4(data_b_v4[b_idx + 1]);
|
| 50 |
+
sum +=
|
| 51 |
+
fma(FLOAT_TYPE(b0.x), FLOAT_TYPE((sign & 1) != 0 ? -grid0.x : grid0.x),
|
| 52 |
+
fma(FLOAT_TYPE(b0.y), FLOAT_TYPE((sign & 2) != 0 ? -grid0.y : grid0.y),
|
| 53 |
+
fma(FLOAT_TYPE(b0.z), FLOAT_TYPE((sign & 4) != 0 ? -grid0.z : grid0.z),
|
| 54 |
+
fma(FLOAT_TYPE(b0.w), FLOAT_TYPE((sign & 8) != 0 ? -grid0.w : grid0.w),
|
| 55 |
+
fma(FLOAT_TYPE(b4.x), FLOAT_TYPE((sign & 16) != 0 ? -grid1.x : grid1.x),
|
| 56 |
+
fma(FLOAT_TYPE(b4.y), FLOAT_TYPE((sign & 32) != 0 ? -grid1.y : grid1.y),
|
| 57 |
+
fma(FLOAT_TYPE(b4.z), FLOAT_TYPE((sign & 64) != 0 ? -grid1.z : grid1.z),
|
| 58 |
+
fma(FLOAT_TYPE(b4.w), FLOAT_TYPE((sign7 & 1) != 0 ? -grid1.w : grid1.w),
|
| 59 |
+
FLOAT_TYPE(0.0)))))))));
|
| 60 |
+
}
|
| 61 |
+
temp[j][n] = fma(FLOAT_TYPE(db_vals[n]), sum, temp[j][n]);
|
| 62 |
+
}
|
| 63 |
+
ibi += num_blocks_per_row;
|
| 64 |
+
}
|
| 65 |
+
}
|
| 66 |
+
|
| 67 |
+
void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
|
| 68 |
+
uint a_offset, b_offset, d_offset;
|
| 69 |
+
get_offsets(a_offset, b_offset, d_offset);
|
| 70 |
+
|
| 71 |
+
const uint num_blocks_per_row = p.ncols / QUANT_K;
|
| 72 |
+
|
| 73 |
+
// 16 threads are used to process each block
|
| 74 |
+
const uint blocks_per_wg = gl_WorkGroupSize.x/16;
|
| 75 |
+
const uint tid = gl_LocalInvocationID.x;
|
| 76 |
+
const uint itid = tid % 16; // 0...15
|
| 77 |
+
const uint ix = tid / 16;
|
| 78 |
+
|
| 79 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 80 |
+
[[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) {
|
| 81 |
+
temp[j][i] = FLOAT_TYPE(0);
|
| 82 |
+
}
|
| 83 |
+
}
|
| 84 |
+
|
| 85 |
+
[[unroll]] for (uint i = ix; i < num_blocks_per_row; i += blocks_per_wg)
|
| 86 |
+
calc_superblock(a_offset, b_offset, itid, i, num_blocks_per_row, first_row, num_rows);
|
| 87 |
+
|
| 88 |
+
reduce_result(temp, d_offset, first_row, num_rows, tid);
|
| 89 |
+
}
|
| 90 |
+
|
| 91 |
+
void main() {
|
| 92 |
+
const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z);
|
| 93 |
+
|
| 94 |
+
init_iq_shmem(gl_WorkGroupSize);
|
| 95 |
+
|
| 96 |
+
// do NUM_ROWS at a time, unless there aren't enough remaining rows
|
| 97 |
+
if (first_row + NUM_ROWS <= p.stride_d) {
|
| 98 |
+
compute_outputs(first_row, NUM_ROWS);
|
| 99 |
+
} else {
|
| 100 |
+
if (first_row >= p.stride_d) {
|
| 101 |
+
return;
|
| 102 |
+
}
|
| 103 |
+
compute_outputs(first_row, p.stride_d - first_row);
|
| 104 |
+
}
|
| 105 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq2_xxs.comp
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
|
| 3 |
+
|
| 4 |
+
#include "mul_mat_vec_base.glsl"
|
| 5 |
+
|
| 6 |
+
layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
|
| 7 |
+
|
| 8 |
+
FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];
|
| 9 |
+
|
| 10 |
+
void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) {
|
| 11 |
+
const uint y_idx = i * QUANT_K + 16 * itid;
|
| 12 |
+
const uint ib32 = itid / 2; // 0..7
|
| 13 |
+
|
| 14 |
+
uint ibi = a_offset + first_row * num_blocks_per_row + i;
|
| 15 |
+
[[unroll]] for (uint n = 0; n < num_rows; ++n) {
|
| 16 |
+
const float d = float(data_a[ibi].d);
|
| 17 |
+
const uint signscale = pack32(u16vec2(
|
| 18 |
+
data_a_packed16[ibi].qs[4 * ib32 + 2],
|
| 19 |
+
data_a_packed16[ibi].qs[4 * ib32 + 3]));
|
| 20 |
+
const float db = d * 0.25 * (0.5 + (signscale >> 28));
|
| 21 |
+
[[unroll]] for (uint l = 0; l < 2; ++l) {
|
| 22 |
+
const uint qs = data_a[ibi].qs[8 * ib32 + 2 * (itid & 1) + l];
|
| 23 |
+
const uint sign = bitfieldExtract(signscale, 7 * int(2 * (itid & 1) + l), 7);
|
| 24 |
+
const uint sign7 = bitCount(sign);
|
| 25 |
+
const vec4 grid0 = vec4(unpack8(iq2xxs_grid[qs].x));
|
| 26 |
+
const vec4 grid1 = vec4(unpack8(iq2xxs_grid[qs].y));
|
| 27 |
+
|
| 28 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 29 |
+
const vec4 b0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 0]);
|
| 30 |
+
const vec4 b4 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 1]);
|
| 31 |
+
|
| 32 |
+
FLOAT_TYPE sum =
|
| 33 |
+
fma(FLOAT_TYPE(b0.x), FLOAT_TYPE((sign & 1) != 0 ? -grid0.x : grid0.x),
|
| 34 |
+
fma(FLOAT_TYPE(b0.y), FLOAT_TYPE((sign & 2) != 0 ? -grid0.y : grid0.y),
|
| 35 |
+
fma(FLOAT_TYPE(b0.z), FLOAT_TYPE((sign & 4) != 0 ? -grid0.z : grid0.z),
|
| 36 |
+
fma(FLOAT_TYPE(b0.w), FLOAT_TYPE((sign & 8) != 0 ? -grid0.w : grid0.w),
|
| 37 |
+
fma(FLOAT_TYPE(b4.x), FLOAT_TYPE((sign & 16) != 0 ? -grid1.x : grid1.x),
|
| 38 |
+
fma(FLOAT_TYPE(b4.y), FLOAT_TYPE((sign & 32) != 0 ? -grid1.y : grid1.y),
|
| 39 |
+
fma(FLOAT_TYPE(b4.z), FLOAT_TYPE((sign & 64) != 0 ? -grid1.z : grid1.z),
|
| 40 |
+
fma(FLOAT_TYPE(b4.w), FLOAT_TYPE((sign7 & 1) != 0 ? -grid1.w : grid1.w),
|
| 41 |
+
FLOAT_TYPE(0.0)))))))));
|
| 42 |
+
temp[j][n] = fma(db, sum, temp[j][n]);
|
| 43 |
+
}
|
| 44 |
+
}
|
| 45 |
+
ibi += num_blocks_per_row;
|
| 46 |
+
}
|
| 47 |
+
}
|
| 48 |
+
|
| 49 |
+
void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
|
| 50 |
+
uint a_offset, b_offset, d_offset;
|
| 51 |
+
get_offsets(a_offset, b_offset, d_offset);
|
| 52 |
+
|
| 53 |
+
const uint num_blocks_per_row = p.ncols / QUANT_K;
|
| 54 |
+
|
| 55 |
+
// 16 threads are used to process each block
|
| 56 |
+
const uint blocks_per_wg = gl_WorkGroupSize.x/16;
|
| 57 |
+
const uint tid = gl_LocalInvocationID.x;
|
| 58 |
+
const uint itid = tid % 16; // 0...15
|
| 59 |
+
const uint ix = tid / 16;
|
| 60 |
+
|
| 61 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 62 |
+
[[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) {
|
| 63 |
+
temp[j][i] = FLOAT_TYPE(0);
|
| 64 |
+
}
|
| 65 |
+
}
|
| 66 |
+
|
| 67 |
+
[[unroll]] for (uint i = ix; i < num_blocks_per_row; i += blocks_per_wg)
|
| 68 |
+
calc_superblock(a_offset, b_offset, itid, i, num_blocks_per_row, first_row, num_rows);
|
| 69 |
+
|
| 70 |
+
reduce_result(temp, d_offset, first_row, num_rows, tid);
|
| 71 |
+
}
|
| 72 |
+
|
| 73 |
+
void main() {
|
| 74 |
+
const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z);
|
| 75 |
+
|
| 76 |
+
init_iq_shmem(gl_WorkGroupSize);
|
| 77 |
+
|
| 78 |
+
// do NUM_ROWS at a time, unless there aren't enough remaining rows
|
| 79 |
+
if (first_row + NUM_ROWS <= p.stride_d) {
|
| 80 |
+
compute_outputs(first_row, NUM_ROWS);
|
| 81 |
+
} else {
|
| 82 |
+
if (first_row >= p.stride_d) {
|
| 83 |
+
return;
|
| 84 |
+
}
|
| 85 |
+
compute_outputs(first_row, p.stride_d - first_row);
|
| 86 |
+
}
|
| 87 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_s.comp
ADDED
|
@@ -0,0 +1,90 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
|
| 3 |
+
|
| 4 |
+
#include "mul_mat_vec_base.glsl"
|
| 5 |
+
|
| 6 |
+
layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
|
| 7 |
+
|
| 8 |
+
FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];
|
| 9 |
+
|
| 10 |
+
void calc_superblock(const uint a_offset, const uint b_offset, const uint ib32, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) {
|
| 11 |
+
const uint y_idx = i * QUANT_K + 32 * ib32;
|
| 12 |
+
|
| 13 |
+
uint ibi = a_offset + first_row * num_blocks_per_row + i;
|
| 14 |
+
[[unroll]] for (uint n = 0; n < num_rows; ++n) {
|
| 15 |
+
const float d = float(data_a[ibi].d);
|
| 16 |
+
const uint scale = (data_a[ibi].scales[ib32/2] >> (4 * (ib32 & 1))) & 0xF;
|
| 17 |
+
const float dscale = d * (1 + 2 * scale);
|
| 18 |
+
const uint qh = data_a[ibi].qh[ib32];
|
| 19 |
+
FLOAT_TYPE sum[NUM_COLS];
|
| 20 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 21 |
+
sum[j] = 0.0;
|
| 22 |
+
}
|
| 23 |
+
[[unroll]] for (uint l = 0; l < 4; ++l) {
|
| 24 |
+
const u8vec2 qs = unpack8(uint32_t(data_a_packed16[ibi].qs[4 * ib32 + l])).xy; // vec4 used due to #12147
|
| 25 |
+
const uint sign = data_a[ibi].signs[4 * ib32 + l];
|
| 26 |
+
const vec4 grid0 = vec4(unpack8(iq3s_grid[qs.x | ((qh << (8 - 2*l)) & 0x100)]));
|
| 27 |
+
const vec4 grid1 = vec4(unpack8(iq3s_grid[qs.y | ((qh << (7 - 2*l)) & 0x100)]));
|
| 28 |
+
|
| 29 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 30 |
+
const vec4 b0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 0]);
|
| 31 |
+
const vec4 b4 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 1]);
|
| 32 |
+
|
| 33 |
+
sum[j] =
|
| 34 |
+
fma(FLOAT_TYPE(b0.x), FLOAT_TYPE((sign & 1) != 0 ? -grid0.x : grid0.x),
|
| 35 |
+
fma(FLOAT_TYPE(b0.y), FLOAT_TYPE((sign & 2) != 0 ? -grid0.y : grid0.y),
|
| 36 |
+
fma(FLOAT_TYPE(b0.z), FLOAT_TYPE((sign & 4) != 0 ? -grid0.z : grid0.z),
|
| 37 |
+
fma(FLOAT_TYPE(b0.w), FLOAT_TYPE((sign & 8) != 0 ? -grid0.w : grid0.w),
|
| 38 |
+
fma(FLOAT_TYPE(b4.x), FLOAT_TYPE((sign & 16) != 0 ? -grid1.x : grid1.x),
|
| 39 |
+
fma(FLOAT_TYPE(b4.y), FLOAT_TYPE((sign & 32) != 0 ? -grid1.y : grid1.y),
|
| 40 |
+
fma(FLOAT_TYPE(b4.z), FLOAT_TYPE((sign & 64) != 0 ? -grid1.z : grid1.z),
|
| 41 |
+
fma(FLOAT_TYPE(b4.w), FLOAT_TYPE((sign & 128) != 0 ? -grid1.w : grid1.w),
|
| 42 |
+
sum[j]))))))));
|
| 43 |
+
}
|
| 44 |
+
}
|
| 45 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 46 |
+
temp[j][n] = fma(dscale, sum[j], temp[j][n]);
|
| 47 |
+
}
|
| 48 |
+
ibi += num_blocks_per_row;
|
| 49 |
+
}
|
| 50 |
+
}
|
| 51 |
+
|
| 52 |
+
void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
|
| 53 |
+
uint a_offset, b_offset, d_offset;
|
| 54 |
+
get_offsets(a_offset, b_offset, d_offset);
|
| 55 |
+
|
| 56 |
+
const uint num_blocks_per_row = p.ncols / QUANT_K;
|
| 57 |
+
|
| 58 |
+
// 8 threads are used to process each block
|
| 59 |
+
const uint blocks_per_wg = gl_WorkGroupSize.x/8;
|
| 60 |
+
const uint tid = gl_LocalInvocationID.x;
|
| 61 |
+
const uint itid = tid % 8; // 0...7
|
| 62 |
+
const uint ix = tid / 8;
|
| 63 |
+
|
| 64 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 65 |
+
[[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) {
|
| 66 |
+
temp[j][i] = FLOAT_TYPE(0);
|
| 67 |
+
}
|
| 68 |
+
}
|
| 69 |
+
|
| 70 |
+
[[unroll]] for (uint i = ix; i < num_blocks_per_row; i += blocks_per_wg)
|
| 71 |
+
calc_superblock(a_offset, b_offset, itid, i, num_blocks_per_row, first_row, num_rows);
|
| 72 |
+
|
| 73 |
+
reduce_result(temp, d_offset, first_row, num_rows, tid);
|
| 74 |
+
}
|
| 75 |
+
|
| 76 |
+
void main() {
|
| 77 |
+
const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z);
|
| 78 |
+
|
| 79 |
+
init_iq_shmem(gl_WorkGroupSize);
|
| 80 |
+
|
| 81 |
+
// do NUM_ROWS at a time, unless there aren't enough remaining rows
|
| 82 |
+
if (first_row + NUM_ROWS <= p.stride_d) {
|
| 83 |
+
compute_outputs(first_row, NUM_ROWS);
|
| 84 |
+
} else {
|
| 85 |
+
if (first_row >= p.stride_d) {
|
| 86 |
+
return;
|
| 87 |
+
}
|
| 88 |
+
compute_outputs(first_row, p.stride_d - first_row);
|
| 89 |
+
}
|
| 90 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_iq3_xxs.comp
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
|
| 3 |
+
|
| 4 |
+
#include "mul_mat_vec_base.glsl"
|
| 5 |
+
|
| 6 |
+
layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
|
| 7 |
+
|
| 8 |
+
FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];
|
| 9 |
+
|
| 10 |
+
void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) {
|
| 11 |
+
const uint y_idx = i * QUANT_K + 16 * itid;
|
| 12 |
+
const uint ib32 = itid / 2; // 0..7
|
| 13 |
+
|
| 14 |
+
uint ibi = a_offset + first_row * num_blocks_per_row + i;
|
| 15 |
+
[[unroll]] for (uint n = 0; n < num_rows; ++n) {
|
| 16 |
+
const float d = float(data_a[ibi].d);
|
| 17 |
+
const uint signscale = pack32(u16vec2(
|
| 18 |
+
data_a_packed16[ibi].qs[QUANT_K / 8 + 2 * ib32],
|
| 19 |
+
data_a_packed16[ibi].qs[QUANT_K / 8 + 2 * ib32 + 1]));
|
| 20 |
+
const float db = d * 0.5 * (0.5 + (signscale >> 28));
|
| 21 |
+
[[unroll]] for (uint l = 0; l < 2; ++l) {
|
| 22 |
+
const uint qs0 = data_a[ibi].qs[8 * ib32 + 4 * (itid & 1) + 2 * l];
|
| 23 |
+
const uint qs1 = data_a[ibi].qs[8 * ib32 + 4 * (itid & 1) + 2 * l + 1];
|
| 24 |
+
const uint sign = bitfieldExtract(signscale, 7 * int(2 * (itid & 1) + l), 7);
|
| 25 |
+
const uint sign7 = bitCount(sign);
|
| 26 |
+
const vec4 grid0 = vec4(unpack8(iq3xxs_grid[qs0]));
|
| 27 |
+
const vec4 grid1 = vec4(unpack8(iq3xxs_grid[qs1]));
|
| 28 |
+
|
| 29 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 30 |
+
const vec4 b0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 0]);
|
| 31 |
+
const vec4 b4 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 2*l + 1]);
|
| 32 |
+
|
| 33 |
+
FLOAT_TYPE sum =
|
| 34 |
+
fma(FLOAT_TYPE(b0.x), FLOAT_TYPE((sign & 1) != 0 ? -grid0.x : grid0.x),
|
| 35 |
+
fma(FLOAT_TYPE(b0.y), FLOAT_TYPE((sign & 2) != 0 ? -grid0.y : grid0.y),
|
| 36 |
+
fma(FLOAT_TYPE(b0.z), FLOAT_TYPE((sign & 4) != 0 ? -grid0.z : grid0.z),
|
| 37 |
+
fma(FLOAT_TYPE(b0.w), FLOAT_TYPE((sign & 8) != 0 ? -grid0.w : grid0.w),
|
| 38 |
+
fma(FLOAT_TYPE(b4.x), FLOAT_TYPE((sign & 16) != 0 ? -grid1.x : grid1.x),
|
| 39 |
+
fma(FLOAT_TYPE(b4.y), FLOAT_TYPE((sign & 32) != 0 ? -grid1.y : grid1.y),
|
| 40 |
+
fma(FLOAT_TYPE(b4.z), FLOAT_TYPE((sign & 64) != 0 ? -grid1.z : grid1.z),
|
| 41 |
+
fma(FLOAT_TYPE(b4.w), FLOAT_TYPE((sign7 & 1) != 0 ? -grid1.w : grid1.w),
|
| 42 |
+
FLOAT_TYPE(0.0)))))))));
|
| 43 |
+
temp[j][n] = fma(db, sum, temp[j][n]);
|
| 44 |
+
}
|
| 45 |
+
}
|
| 46 |
+
ibi += num_blocks_per_row;
|
| 47 |
+
}
|
| 48 |
+
}
|
| 49 |
+
|
| 50 |
+
void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
|
| 51 |
+
uint a_offset, b_offset, d_offset;
|
| 52 |
+
get_offsets(a_offset, b_offset, d_offset);
|
| 53 |
+
|
| 54 |
+
const uint num_blocks_per_row = p.ncols / QUANT_K;
|
| 55 |
+
|
| 56 |
+
// 16 threads are used to process each block
|
| 57 |
+
const uint blocks_per_wg = gl_WorkGroupSize.x/16;
|
| 58 |
+
const uint tid = gl_LocalInvocationID.x;
|
| 59 |
+
const uint itid = tid % 16; // 0...15
|
| 60 |
+
const uint ix = tid / 16;
|
| 61 |
+
|
| 62 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 63 |
+
[[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) {
|
| 64 |
+
temp[j][i] = FLOAT_TYPE(0);
|
| 65 |
+
}
|
| 66 |
+
}
|
| 67 |
+
|
| 68 |
+
[[unroll]] for (uint i = ix; i < num_blocks_per_row; i += blocks_per_wg)
|
| 69 |
+
calc_superblock(a_offset, b_offset, itid, i, num_blocks_per_row, first_row, num_rows);
|
| 70 |
+
|
| 71 |
+
reduce_result(temp, d_offset, first_row, num_rows, tid);
|
| 72 |
+
}
|
| 73 |
+
|
| 74 |
+
void main() {
|
| 75 |
+
const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z);
|
| 76 |
+
|
| 77 |
+
init_iq_shmem(gl_WorkGroupSize);
|
| 78 |
+
|
| 79 |
+
// do NUM_ROWS at a time, unless there aren't enough remaining rows
|
| 80 |
+
if (first_row + NUM_ROWS <= p.stride_d) {
|
| 81 |
+
compute_outputs(first_row, NUM_ROWS);
|
| 82 |
+
} else {
|
| 83 |
+
if (first_row >= p.stride_d) {
|
| 84 |
+
return;
|
| 85 |
+
}
|
| 86 |
+
compute_outputs(first_row, p.stride_d - first_row);
|
| 87 |
+
}
|
| 88 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_nc.comp
ADDED
|
@@ -0,0 +1,124 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
|
| 3 |
+
#extension GL_EXT_control_flow_attributes : enable
|
| 4 |
+
#extension GL_EXT_shader_16bit_storage : require
|
| 5 |
+
|
| 6 |
+
#define BLOCK_SIZE 32
|
| 7 |
+
#define FLOAT_TYPE float
|
| 8 |
+
|
| 9 |
+
layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in;
|
| 10 |
+
|
| 11 |
+
#include "mul_mat_vec_iface.glsl"
|
| 12 |
+
|
| 13 |
+
layout (push_constant) uniform parameter
|
| 14 |
+
{
|
| 15 |
+
uint ncols_x;
|
| 16 |
+
uint nrows_x;
|
| 17 |
+
uint row_stride_x;
|
| 18 |
+
uint channel_stride_x;
|
| 19 |
+
uint channel_stride_y;
|
| 20 |
+
uint channel_x_divisor;
|
| 21 |
+
uint ne12;
|
| 22 |
+
uint b_offset;
|
| 23 |
+
uint d_offset;
|
| 24 |
+
uint nb03;
|
| 25 |
+
uint nb13;
|
| 26 |
+
uint nb23;
|
| 27 |
+
uint fusion_flags;
|
| 28 |
+
} p;
|
| 29 |
+
|
| 30 |
+
shared FLOAT_TYPE tmp[BLOCK_SIZE];
|
| 31 |
+
|
| 32 |
+
void main() {
|
| 33 |
+
const uint tid = gl_LocalInvocationID.x;
|
| 34 |
+
const uint row_x = gl_GlobalInvocationID.y;
|
| 35 |
+
const uint channel = gl_GlobalInvocationID.z;
|
| 36 |
+
const uint i3 = gl_WorkGroupID.x;
|
| 37 |
+
const uint channel_x = channel / p.channel_x_divisor;
|
| 38 |
+
const uint channel_y = channel % p.ne12;
|
| 39 |
+
|
| 40 |
+
const uint nrows_y = p.ncols_x;
|
| 41 |
+
const uint nrows_dst = p.nrows_x;
|
| 42 |
+
const uint row_dst = row_x;
|
| 43 |
+
|
| 44 |
+
const uint idst = i3*p.nb23 + channel*nrows_dst + row_dst;
|
| 45 |
+
|
| 46 |
+
FLOAT_TYPE temp = 0.0f;
|
| 47 |
+
|
| 48 |
+
// Detect alignment for vector loads
|
| 49 |
+
bool is_aligned = (p.ncols_x % 4) == 0 && (p.row_stride_x % 4) == 0 && (p.channel_stride_x % 4) == 0;
|
| 50 |
+
|
| 51 |
+
for (uint col_x0 = 0; col_x0 < p.ncols_x;) {
|
| 52 |
+
|
| 53 |
+
// Unroll 2x and do vec4 loads if aligned
|
| 54 |
+
const uint unroll_count = 2;
|
| 55 |
+
if (col_x0 + unroll_count * 4 * BLOCK_SIZE <= p.ncols_x && is_aligned) {
|
| 56 |
+
[[unroll]] for (uint i = 0; i < unroll_count; ++i) {
|
| 57 |
+
const uint col_x = col_x0 + 4*tid;
|
| 58 |
+
|
| 59 |
+
const uint row_y = col_x;
|
| 60 |
+
|
| 61 |
+
const uint ix = i3*p.nb03 + channel_x*p.channel_stride_x + row_x*p.row_stride_x + col_x;
|
| 62 |
+
const uint iy = i3*p.nb13 + channel_y*p.channel_stride_y + row_y;
|
| 63 |
+
|
| 64 |
+
const vec4 av4 = vec4(data_a_v4[ix / 4]);
|
| 65 |
+
const vec4 bv4 = vec4(data_b_v4[iy / 4]);
|
| 66 |
+
|
| 67 |
+
temp += dot(av4, bv4);
|
| 68 |
+
|
| 69 |
+
col_x0 += 4*BLOCK_SIZE;
|
| 70 |
+
}
|
| 71 |
+
// do vec4 loads if aligned
|
| 72 |
+
} else if (col_x0 + 4*BLOCK_SIZE <= p.ncols_x && is_aligned) {
|
| 73 |
+
const uint col_x = col_x0 + 4*tid;
|
| 74 |
+
|
| 75 |
+
const uint row_y = col_x;
|
| 76 |
+
|
| 77 |
+
const uint ix = i3*p.nb03 + channel_x*p.channel_stride_x + row_x*p.row_stride_x + col_x;
|
| 78 |
+
const uint iy = i3*p.nb13 + channel_y*p.channel_stride_y + row_y;
|
| 79 |
+
|
| 80 |
+
const vec4 av4 = vec4(data_a_v4[ix / 4]);
|
| 81 |
+
const vec4 bv4 = vec4(data_b_v4[iy / 4]);
|
| 82 |
+
|
| 83 |
+
temp += dot(av4, bv4);
|
| 84 |
+
|
| 85 |
+
col_x0 += 4*BLOCK_SIZE;
|
| 86 |
+
} else {
|
| 87 |
+
const uint col_x = col_x0 + tid;
|
| 88 |
+
if (col_x >= p.ncols_x) {
|
| 89 |
+
break;
|
| 90 |
+
}
|
| 91 |
+
|
| 92 |
+
const uint row_y = col_x;
|
| 93 |
+
|
| 94 |
+
const uint ix = i3*p.nb03 + channel_x*p.channel_stride_x + row_x*p.row_stride_x + col_x;
|
| 95 |
+
const uint iy = i3*p.nb13 + channel_y*p.channel_stride_y + row_y;
|
| 96 |
+
|
| 97 |
+
const FLOAT_TYPE xi = FLOAT_TYPE(data_a[ix]);
|
| 98 |
+
|
| 99 |
+
temp = fma(xi, FLOAT_TYPE(data_b[iy]), temp);
|
| 100 |
+
col_x0 += BLOCK_SIZE;
|
| 101 |
+
}
|
| 102 |
+
}
|
| 103 |
+
|
| 104 |
+
tmp[tid] = temp;
|
| 105 |
+
|
| 106 |
+
// sum up partial sums and write back result
|
| 107 |
+
barrier();
|
| 108 |
+
[[unroll]] for (int s = BLOCK_SIZE / 2; s > 0; s >>= 1) {
|
| 109 |
+
if (tid < s) {
|
| 110 |
+
tmp[tid] += tmp[tid + s];
|
| 111 |
+
}
|
| 112 |
+
barrier();
|
| 113 |
+
}
|
| 114 |
+
|
| 115 |
+
if (tid == 0) {
|
| 116 |
+
if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0) {
|
| 117 |
+
tmp[0] += FLOAT_TYPE(data_fuse0[idst]);
|
| 118 |
+
}
|
| 119 |
+
if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS1) != 0) {
|
| 120 |
+
tmp[0] += FLOAT_TYPE(data_fuse1[idst]);
|
| 121 |
+
}
|
| 122 |
+
data_d[idst] = tmp[0];
|
| 123 |
+
}
|
| 124 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_p021.comp
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
|
| 3 |
+
#extension GL_EXT_control_flow_attributes : enable
|
| 4 |
+
#extension GL_EXT_shader_16bit_storage : require
|
| 5 |
+
#if USE_SUBGROUP_ADD
|
| 6 |
+
#extension GL_KHR_shader_subgroup_arithmetic : enable
|
| 7 |
+
#endif
|
| 8 |
+
|
| 9 |
+
#define FLOAT_TYPE float
|
| 10 |
+
|
| 11 |
+
layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
|
| 12 |
+
|
| 13 |
+
#include "mul_mat_vec_iface.glsl"
|
| 14 |
+
|
| 15 |
+
layout(constant_id = 0) const int BLOCK_SIZE = 32;
|
| 16 |
+
// gqa_ratio is in the range [1,8]
|
| 17 |
+
layout(constant_id = 1) const uint gqa_ratio = 1;
|
| 18 |
+
|
| 19 |
+
layout (push_constant) uniform parameter
|
| 20 |
+
{
|
| 21 |
+
uint ncols_x;
|
| 22 |
+
uint nrows_x;
|
| 23 |
+
uint nchannels_x;
|
| 24 |
+
uint nchannels_y;
|
| 25 |
+
uint b_offset;
|
| 26 |
+
uint d_offset;
|
| 27 |
+
uint fusion_flags;
|
| 28 |
+
} p;
|
| 29 |
+
|
| 30 |
+
#if !USE_SUBGROUP_ADD
|
| 31 |
+
shared FLOAT_TYPE tmp[8][BLOCK_SIZE];
|
| 32 |
+
#endif
|
| 33 |
+
|
| 34 |
+
void main() {
|
| 35 |
+
const uint tid = gl_LocalInvocationID.x;
|
| 36 |
+
const uint row_x = gl_GlobalInvocationID.y;
|
| 37 |
+
|
| 38 |
+
uint channel, channel_x;
|
| 39 |
+
|
| 40 |
+
// When gqa_ratio > 1, each invocation does multiple rows.
|
| 41 |
+
// The row in the A matrix is starting from channel / gqa_ratio and the
|
| 42 |
+
// rows in the B matrix are [channel, channel+gqa_ratio).
|
| 43 |
+
// When gpa_ratio is 1, each invocation does one row.
|
| 44 |
+
if (gqa_ratio > 1) {
|
| 45 |
+
channel_x = gl_GlobalInvocationID.z;
|
| 46 |
+
channel = channel_x * gqa_ratio;
|
| 47 |
+
} else {
|
| 48 |
+
channel = gl_GlobalInvocationID.z;
|
| 49 |
+
channel_x = channel / (p.nchannels_y / p.nchannels_x);;
|
| 50 |
+
}
|
| 51 |
+
|
| 52 |
+
const uint nrows_y = p.ncols_x;
|
| 53 |
+
const uint nrows_dst = p.nrows_x;
|
| 54 |
+
const uint row_dst = row_x;
|
| 55 |
+
|
| 56 |
+
FLOAT_TYPE temp[8];
|
| 57 |
+
[[unroll]] for (uint i = 0; i < 8; ++i) {
|
| 58 |
+
temp[i] = FLOAT_TYPE(0.0f);
|
| 59 |
+
}
|
| 60 |
+
|
| 61 |
+
// Detect alignment for vector loads
|
| 62 |
+
bool is_aligned = (p.ncols_x % 4) == 0 && (p.nchannels_x % 4) == 0 && (nrows_y % 4) == 0;
|
| 63 |
+
|
| 64 |
+
for (uint col_x0 = 0; col_x0 < p.ncols_x; col_x0 += BLOCK_SIZE) {
|
| 65 |
+
|
| 66 |
+
// Use vec4 loads if aligned
|
| 67 |
+
if (col_x0 + 4*BLOCK_SIZE <= p.ncols_x && is_aligned) {
|
| 68 |
+
|
| 69 |
+
uint col_x = col_x0 + 4*tid;
|
| 70 |
+
const uint row_y = col_x;
|
| 71 |
+
|
| 72 |
+
// x is transposed and permuted
|
| 73 |
+
const uint ix = row_x*p.nchannels_x*p.ncols_x + channel_x*p.ncols_x + col_x;
|
| 74 |
+
const vec4 av4 = vec4(data_a_v4[ix / 4]);
|
| 75 |
+
|
| 76 |
+
[[unroll]] for (uint c = 0; c < gqa_ratio; ++c) {
|
| 77 |
+
// y is not transposed but permuted
|
| 78 |
+
const uint iy = (channel + c)*nrows_y + row_y;
|
| 79 |
+
|
| 80 |
+
vec4 bv4 = data_b_v4[iy / 4];
|
| 81 |
+
temp[c] += dot(av4, bv4);
|
| 82 |
+
}
|
| 83 |
+
|
| 84 |
+
col_x0 += 3*BLOCK_SIZE;
|
| 85 |
+
} else {
|
| 86 |
+
const uint col_x = col_x0 + tid;
|
| 87 |
+
|
| 88 |
+
if (col_x >= p.ncols_x) {
|
| 89 |
+
break;
|
| 90 |
+
}
|
| 91 |
+
|
| 92 |
+
// x is transposed and permuted
|
| 93 |
+
const uint ix = row_x*p.nchannels_x*p.ncols_x + channel_x*p.ncols_x + col_x;
|
| 94 |
+
const FLOAT_TYPE xi = FLOAT_TYPE(data_a[ix]);
|
| 95 |
+
|
| 96 |
+
const uint row_y = col_x;
|
| 97 |
+
|
| 98 |
+
[[unroll]] for (uint c = 0; c < gqa_ratio; ++c) {
|
| 99 |
+
// y is not transposed but permuted
|
| 100 |
+
const uint iy = (channel + c)*nrows_y + row_y;
|
| 101 |
+
|
| 102 |
+
temp[c] = fma(xi, FLOAT_TYPE(data_b[iy]), temp[c]);
|
| 103 |
+
}
|
| 104 |
+
}
|
| 105 |
+
}
|
| 106 |
+
|
| 107 |
+
#if USE_SUBGROUP_ADD
|
| 108 |
+
// reduce vec4 at a time
|
| 109 |
+
vec4 t = vec4(temp[0], temp[1], temp[2], temp[3]);
|
| 110 |
+
t = subgroupAdd(t);
|
| 111 |
+
temp[0] = t[0];
|
| 112 |
+
temp[1] = t[1];
|
| 113 |
+
temp[2] = t[2];
|
| 114 |
+
temp[3] = t[3];
|
| 115 |
+
if (gqa_ratio > 4) {
|
| 116 |
+
t = vec4(temp[4], temp[5], temp[6], temp[7]);
|
| 117 |
+
t = subgroupAdd(t);
|
| 118 |
+
temp[4] = t[0];
|
| 119 |
+
temp[5] = t[1];
|
| 120 |
+
temp[6] = t[2];
|
| 121 |
+
temp[7] = t[3];
|
| 122 |
+
}
|
| 123 |
+
#else
|
| 124 |
+
[[unroll]] for (uint c = 0; c < gqa_ratio; ++c) {
|
| 125 |
+
tmp[c][tid] = temp[c];
|
| 126 |
+
}
|
| 127 |
+
// sum up partial sums and write back result
|
| 128 |
+
barrier();
|
| 129 |
+
[[unroll]] for (int s = BLOCK_SIZE / 2; s > 0; s >>= 1) {
|
| 130 |
+
if (tid < s) {
|
| 131 |
+
[[unroll]] for (uint c = 0; c < gqa_ratio; ++c) {
|
| 132 |
+
temp[c] += tmp[c][tid + s];
|
| 133 |
+
tmp[c][tid] = temp[c];
|
| 134 |
+
}
|
| 135 |
+
}
|
| 136 |
+
barrier();
|
| 137 |
+
}
|
| 138 |
+
[[unroll]] for (uint c = 0; c < gqa_ratio; ++c) {
|
| 139 |
+
temp[c] = tmp[c][tid];
|
| 140 |
+
}
|
| 141 |
+
#endif
|
| 142 |
+
|
| 143 |
+
if (tid == 0) {
|
| 144 |
+
[[unroll]] for (uint c = 0; c < gqa_ratio; ++c) {
|
| 145 |
+
// dst is not transposed and not permuted
|
| 146 |
+
const uint idst = (channel + c)*nrows_dst + row_dst;
|
| 147 |
+
if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS0) != 0) {
|
| 148 |
+
temp[c] += FLOAT_TYPE(data_fuse0[idst]);
|
| 149 |
+
}
|
| 150 |
+
if ((p.fusion_flags & MAT_VEC_FUSION_FLAGS_BIAS1) != 0) {
|
| 151 |
+
temp[c] += FLOAT_TYPE(data_fuse1[idst]);
|
| 152 |
+
}
|
| 153 |
+
data_d[idst] = temp[c];
|
| 154 |
+
}
|
| 155 |
+
}
|
| 156 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q2_k.comp
ADDED
|
@@ -0,0 +1,128 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
|
| 3 |
+
|
| 4 |
+
#include "mul_mat_vec_base.glsl"
|
| 5 |
+
|
| 6 |
+
layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
|
| 7 |
+
|
| 8 |
+
shared FLOAT_TYPE sccache1[2][BLOCK_SIZE/16][16];
|
| 9 |
+
shared FLOAT_TYPE sccache2[2][BLOCK_SIZE/16][16];
|
| 10 |
+
|
| 11 |
+
FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];
|
| 12 |
+
uint csel = 0;
|
| 13 |
+
|
| 14 |
+
void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint v_im, const uint ix, const uint q_offset, const uint y_offset, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows, const bool all_threads) {
|
| 15 |
+
const uint y_idx = i * QUANT_K + y_offset;
|
| 16 |
+
|
| 17 |
+
[[unroll]] for (uint n = 0; n < num_rows; ++n) {
|
| 18 |
+
const uint ib0 = a_offset + (first_row+n)*num_blocks_per_row;
|
| 19 |
+
csel ^= 1;
|
| 20 |
+
|
| 21 |
+
if (!all_threads) { // when we don't have enough blocks to use all threads
|
| 22 |
+
if (i < num_blocks_per_row) {
|
| 23 |
+
const uint32_t scale = uint32_t(data_a[ib0 + i].scales[itid]);
|
| 24 |
+
sccache1[csel][ix][itid] = FLOAT_TYPE(scale & 0xF);
|
| 25 |
+
sccache2[csel][ix][itid] = FLOAT_TYPE((scale >> 4) & 0xF);
|
| 26 |
+
}
|
| 27 |
+
barrier();
|
| 28 |
+
|
| 29 |
+
if (i >= num_blocks_per_row)
|
| 30 |
+
continue;
|
| 31 |
+
} else {
|
| 32 |
+
const uint32_t scale = uint32_t(data_a[ib0 + i].scales[itid]);
|
| 33 |
+
sccache1[csel][ix][itid] = FLOAT_TYPE(scale & 0xF);
|
| 34 |
+
sccache2[csel][ix][itid] = FLOAT_TYPE((scale >> 4) & 0xF);
|
| 35 |
+
barrier();
|
| 36 |
+
}
|
| 37 |
+
|
| 38 |
+
const uint32_t qs_u32 = uint32_t(data_a_packed16[ib0 + i].qs[q_offset / 2]) | (uint32_t(data_a_packed16[ib0 + i].qs[q_offset / 2 + 8]) << 16);
|
| 39 |
+
const vec4 qs_u32_0 = vec4(unpack8(qs_u32 & 0x03030303));
|
| 40 |
+
const vec4 qs_u32_2 = vec4(unpack8((qs_u32 >> 2) & 0x03030303));
|
| 41 |
+
const vec4 qs_u32_4 = vec4(unpack8((qs_u32 >> 4) & 0x03030303));
|
| 42 |
+
const vec4 qs_u32_6 = vec4(unpack8((qs_u32 >> 6) & 0x03030303));
|
| 43 |
+
|
| 44 |
+
const FLOAT_TYPEV2 dm = vec2(data_a[ib0 + i].dm);
|
| 45 |
+
|
| 46 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 47 |
+
vec2 b0 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 0]);
|
| 48 |
+
vec2 b16 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 8]);
|
| 49 |
+
vec2 b32 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 16]);
|
| 50 |
+
vec2 b48 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 24]);
|
| 51 |
+
vec2 b64 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 32]);
|
| 52 |
+
vec2 b80 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 40]);
|
| 53 |
+
vec2 b96 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 48]);
|
| 54 |
+
vec2 b112 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 56]);
|
| 55 |
+
|
| 56 |
+
FLOAT_TYPE sum1 = FLOAT_TYPE(0.0);
|
| 57 |
+
FLOAT_TYPE sum2 = FLOAT_TYPE(0.0);
|
| 58 |
+
[[unroll]] for (int l = 0; l < 2; ++l) {
|
| 59 |
+
sum1 = fma(FLOAT_TYPE(b0[l]), sccache1[csel][ix][ 8*v_im] * qs_u32_0[l ],
|
| 60 |
+
fma(FLOAT_TYPE(b16[l]), sccache1[csel][ix][1 + 8*v_im] * qs_u32_0[l+2],
|
| 61 |
+
fma(FLOAT_TYPE(b32[l]), sccache1[csel][ix][2 + 8*v_im] * qs_u32_2[l ],
|
| 62 |
+
fma(FLOAT_TYPE(b48[l]), sccache1[csel][ix][3 + 8*v_im] * qs_u32_2[l+2],
|
| 63 |
+
fma(FLOAT_TYPE(b64[l]), sccache1[csel][ix][4 + 8*v_im] * qs_u32_4[l ],
|
| 64 |
+
fma(FLOAT_TYPE(b80[l]), sccache1[csel][ix][5 + 8*v_im] * qs_u32_4[l+2],
|
| 65 |
+
fma(FLOAT_TYPE(b96[l]), sccache1[csel][ix][6 + 8*v_im] * qs_u32_6[l ],
|
| 66 |
+
fma(FLOAT_TYPE(b112[l]), sccache1[csel][ix][7 + 8*v_im] * qs_u32_6[l+2], sum1))))))));
|
| 67 |
+
sum2 = fma(FLOAT_TYPE(b0[l]), sccache2[csel][ix][ 8*v_im],
|
| 68 |
+
fma(FLOAT_TYPE(b16[l]), sccache2[csel][ix][1 + 8*v_im],
|
| 69 |
+
fma(FLOAT_TYPE(b32[l]), sccache2[csel][ix][2 + 8*v_im],
|
| 70 |
+
fma(FLOAT_TYPE(b48[l]), sccache2[csel][ix][3 + 8*v_im],
|
| 71 |
+
fma(FLOAT_TYPE(b64[l]), sccache2[csel][ix][4 + 8*v_im],
|
| 72 |
+
fma(FLOAT_TYPE(b80[l]), sccache2[csel][ix][5 + 8*v_im],
|
| 73 |
+
fma(FLOAT_TYPE(b96[l]), sccache2[csel][ix][6 + 8*v_im],
|
| 74 |
+
fma(FLOAT_TYPE(b112[l]), sccache2[csel][ix][7 + 8*v_im], sum2))))))));
|
| 75 |
+
}
|
| 76 |
+
temp[j][n] = fma(dm.x, sum1, fma(-dm.y, sum2, temp[j][n]));
|
| 77 |
+
}
|
| 78 |
+
}
|
| 79 |
+
}
|
| 80 |
+
|
| 81 |
+
void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
|
| 82 |
+
uint a_offset, b_offset, d_offset;
|
| 83 |
+
get_offsets(a_offset, b_offset, d_offset);
|
| 84 |
+
|
| 85 |
+
const uint num_blocks_per_row = p.ncols / QUANT_K;
|
| 86 |
+
|
| 87 |
+
// 16 threads are used to process each block
|
| 88 |
+
const uint it_size = gl_WorkGroupSize.x/16;
|
| 89 |
+
const uint tid = gl_LocalInvocationID.x;
|
| 90 |
+
const uint itid = tid%16; // 0...15
|
| 91 |
+
const uint ix = tid/16;
|
| 92 |
+
|
| 93 |
+
const uint v_im = itid/8; // 0 or 1. 0 computes 0..., 1 computes 128...
|
| 94 |
+
const uint v_in = itid - 8*v_im; // 0...7
|
| 95 |
+
|
| 96 |
+
const uint l0 = 2*v_in; // 0...15
|
| 97 |
+
const uint q_offset = 32*v_im + l0;
|
| 98 |
+
const uint y_offset = 128*v_im + l0;
|
| 99 |
+
|
| 100 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 101 |
+
[[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) {
|
| 102 |
+
temp[j][i] = FLOAT_TYPE(0);
|
| 103 |
+
}
|
| 104 |
+
}
|
| 105 |
+
|
| 106 |
+
const uint nbr_par_th = num_blocks_per_row%it_size;
|
| 107 |
+
const uint nbr_all_th = num_blocks_per_row - nbr_par_th;
|
| 108 |
+
uint i0 = 0;
|
| 109 |
+
[[unroll]] for (; i0 < nbr_all_th; i0 += it_size)
|
| 110 |
+
calc_superblock(a_offset, b_offset, itid, v_im, ix, q_offset, y_offset, i0 + ix, num_blocks_per_row, first_row, num_rows, true);
|
| 111 |
+
calc_superblock(a_offset, b_offset, itid, v_im, ix, q_offset, y_offset, i0 + ix, num_blocks_per_row, first_row, num_rows, false);
|
| 112 |
+
|
| 113 |
+
reduce_result(temp, d_offset, first_row, num_rows, tid);
|
| 114 |
+
}
|
| 115 |
+
|
| 116 |
+
void main() {
|
| 117 |
+
const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z);
|
| 118 |
+
|
| 119 |
+
// do NUM_ROWS at a time, unless there aren't enough remaining rows
|
| 120 |
+
if (first_row + NUM_ROWS <= p.stride_d) {
|
| 121 |
+
compute_outputs(first_row, NUM_ROWS);
|
| 122 |
+
} else {
|
| 123 |
+
if (first_row >= p.stride_d) {
|
| 124 |
+
return;
|
| 125 |
+
}
|
| 126 |
+
compute_outputs(first_row, p.stride_d - first_row);
|
| 127 |
+
}
|
| 128 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q3_k.comp
ADDED
|
@@ -0,0 +1,132 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
|
| 3 |
+
|
| 4 |
+
#include "mul_mat_vec_base.glsl"
|
| 5 |
+
|
| 6 |
+
layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
|
| 7 |
+
|
| 8 |
+
shared FLOAT_TYPE sccache[2][BLOCK_SIZE/16][2][8];
|
| 9 |
+
|
| 10 |
+
FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];
|
| 11 |
+
uint csel = 0;
|
| 12 |
+
|
| 13 |
+
void calc_superblock(const uint a_offset, const uint b_offset, const uint ix, const uint itid8, const uint v_im, const uint v_im4, const uint v_in, const uint32_t hm_m[4], const uint q_offset, const uint y_offset, const uint s_shift, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows, const bool all_threads) {
|
| 14 |
+
const uint y_idx = i * QUANT_K + y_offset;
|
| 15 |
+
|
| 16 |
+
[[unroll]] for (uint n = 0; n < num_rows; ++n) {
|
| 17 |
+
const uint ib0 = a_offset + (first_row+n)*num_blocks_per_row;
|
| 18 |
+
csel ^= 1;
|
| 19 |
+
|
| 20 |
+
if (!all_threads) { // when we don't have enough blocks to use all threads
|
| 21 |
+
if (i < num_blocks_per_row)
|
| 22 |
+
sccache[csel][ix][v_im][itid8] = FLOAT_TYPE(int8_t(((data_a[ib0+i].scales[itid8] >> v_im4) & 0xF) | (((data_a[ib0+i].scales[itid8%4+8] >> s_shift) & 3) << 4)) - 32);
|
| 23 |
+
barrier();
|
| 24 |
+
|
| 25 |
+
if (i >= num_blocks_per_row)
|
| 26 |
+
continue;
|
| 27 |
+
}
|
| 28 |
+
|
| 29 |
+
const uint32_t hmk = ~(uint32_t(data_a_packed16[ib0 + i].hmask[v_in]) | (uint32_t(data_a_packed16[ib0 + i].hmask[v_in + 8]) << 16));
|
| 30 |
+
const vec4 hmk_0 = vec4(unpack8(((hmk & hm_m[0]) >> ( v_im4)) << 2));
|
| 31 |
+
const vec4 hmk_1 = vec4(unpack8(((hmk & hm_m[1]) >> (1 + v_im4)) << 2));
|
| 32 |
+
const vec4 hmk_2 = vec4(unpack8(((hmk & hm_m[2]) >> (2 + v_im4)) << 2));
|
| 33 |
+
const vec4 hmk_3 = vec4(unpack8(((hmk & hm_m[3]) >> (3 + v_im4)) << 2));
|
| 34 |
+
|
| 35 |
+
// 0, 1, 16, 17
|
| 36 |
+
uint32_t qs_u32 = uint32_t(data_a[ib0 + i].qs[q_offset]) | (uint32_t(data_a[ib0 + i].qs[q_offset + 1]) << 8);
|
| 37 |
+
qs_u32 |= (uint32_t(data_a[ib0 + i].qs[q_offset + 16]) | (uint32_t(data_a[ib0 + i].qs[q_offset + 17]) << 8)) << 16;
|
| 38 |
+
const vec4 qs_u32_0 = vec4(unpack8(qs_u32 & 0x03030303));
|
| 39 |
+
const vec4 qs_u32_2 = vec4(unpack8((qs_u32 >> 2) & 0x03030303));
|
| 40 |
+
const vec4 qs_u32_4 = vec4(unpack8((qs_u32 >> 4) & 0x03030303));
|
| 41 |
+
const vec4 qs_u32_6 = vec4(unpack8((qs_u32 >> 6) & 0x03030303));
|
| 42 |
+
|
| 43 |
+
if (all_threads) {
|
| 44 |
+
sccache[csel][ix][v_im][itid8] = FLOAT_TYPE(int8_t(((data_a[ib0+i].scales[itid8] >> v_im4) & 0xF) | (((data_a[ib0+i].scales[itid8%4+8] >> s_shift) & 3) << 4)) - 32);
|
| 45 |
+
barrier();
|
| 46 |
+
}
|
| 47 |
+
|
| 48 |
+
const FLOAT_TYPE d = FLOAT_TYPE(data_a[ib0 + i].d);
|
| 49 |
+
|
| 50 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 51 |
+
vec2 b0 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 0]);
|
| 52 |
+
vec2 b16 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 8]);
|
| 53 |
+
vec2 b32 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 16]);
|
| 54 |
+
vec2 b48 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 24]);
|
| 55 |
+
vec2 b64 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 32]);
|
| 56 |
+
vec2 b80 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 40]);
|
| 57 |
+
vec2 b96 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 48]);
|
| 58 |
+
vec2 b112 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y_idx) / 2 + 56]);
|
| 59 |
+
|
| 60 |
+
FLOAT_TYPE sum = FLOAT_TYPE(0.0);
|
| 61 |
+
[[unroll]] for (int l = 0; l < 2; ++l) {
|
| 62 |
+
sum = fma(FLOAT_TYPE( b0[l]) * sccache[csel][ix][v_im][0], qs_u32_0[l ] - hmk_0[l ],
|
| 63 |
+
fma(FLOAT_TYPE( b16[l]) * sccache[csel][ix][v_im][1], qs_u32_0[l+2] - hmk_0[l+2],
|
| 64 |
+
fma(FLOAT_TYPE( b32[l]) * sccache[csel][ix][v_im][2], qs_u32_2[l ] - hmk_1[l ],
|
| 65 |
+
fma(FLOAT_TYPE( b48[l]) * sccache[csel][ix][v_im][3], qs_u32_2[l+2] - hmk_1[l+2],
|
| 66 |
+
fma(FLOAT_TYPE( b64[l]) * sccache[csel][ix][v_im][4], qs_u32_4[l ] - hmk_2[l ],
|
| 67 |
+
fma(FLOAT_TYPE( b80[l]) * sccache[csel][ix][v_im][5], qs_u32_4[l+2] - hmk_2[l+2],
|
| 68 |
+
fma(FLOAT_TYPE( b96[l]) * sccache[csel][ix][v_im][6], qs_u32_6[l ] - hmk_3[l ],
|
| 69 |
+
fma(FLOAT_TYPE(b112[l]) * sccache[csel][ix][v_im][7], qs_u32_6[l+2] - hmk_3[l+2], sum))))))));
|
| 70 |
+
}
|
| 71 |
+
temp[j][n] = fma(d, sum, temp[j][n]);
|
| 72 |
+
}
|
| 73 |
+
}
|
| 74 |
+
}
|
| 75 |
+
|
| 76 |
+
void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
|
| 77 |
+
uint a_offset, b_offset, d_offset;
|
| 78 |
+
get_offsets(a_offset, b_offset, d_offset);
|
| 79 |
+
|
| 80 |
+
const uint num_blocks_per_row = p.ncols / QUANT_K;
|
| 81 |
+
|
| 82 |
+
// 16 threads are used to process each block
|
| 83 |
+
const uint it_size = gl_WorkGroupSize.x/16;
|
| 84 |
+
const uint tid = gl_LocalInvocationID.x;
|
| 85 |
+
const uint itid = tid%16; // 0...15
|
| 86 |
+
const uint ix = tid/16;
|
| 87 |
+
const uint itid8 = itid%8;
|
| 88 |
+
|
| 89 |
+
const uint v_im = itid/8; // 0 or 1. 0 computes 0..., 1 computes 128...
|
| 90 |
+
const uint v_im4 = v_im*4;
|
| 91 |
+
const uint v_in = itid - 8*v_im; // 0...7
|
| 92 |
+
|
| 93 |
+
const uint32_t m = 0x01010101 << (4 * v_im);
|
| 94 |
+
uint32_t hm_m[4];
|
| 95 |
+
[[unroll]] for (uint j = 0; j < 4; ++j)
|
| 96 |
+
hm_m[j] = m << j;
|
| 97 |
+
|
| 98 |
+
const uint l0 = 2*v_in; // 0...15
|
| 99 |
+
const uint q_offset = 32*v_im + l0;
|
| 100 |
+
const uint y_offset = 128*v_im + l0;
|
| 101 |
+
|
| 102 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 103 |
+
[[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) {
|
| 104 |
+
temp[j][i] = FLOAT_TYPE(0);
|
| 105 |
+
}
|
| 106 |
+
}
|
| 107 |
+
|
| 108 |
+
const uint s_shift = v_im4 + 2*(itid8/4);
|
| 109 |
+
|
| 110 |
+
const uint nbr_par_th = num_blocks_per_row%it_size;
|
| 111 |
+
const uint nbr_all_th = num_blocks_per_row - nbr_par_th;
|
| 112 |
+
uint i0 = 0;
|
| 113 |
+
[[unroll]] for (; i0 < nbr_all_th; i0 += it_size)
|
| 114 |
+
calc_superblock(a_offset, b_offset, ix, itid8, v_im, v_im4, v_in, hm_m, q_offset, y_offset, s_shift, i0 + ix, num_blocks_per_row, first_row, num_rows, true);
|
| 115 |
+
calc_superblock(a_offset, b_offset, ix, itid8, v_im, v_im4, v_in, hm_m, q_offset, y_offset, s_shift, i0 + ix, num_blocks_per_row, first_row, num_rows, false);
|
| 116 |
+
|
| 117 |
+
reduce_result(temp, d_offset, first_row, num_rows, tid);
|
| 118 |
+
}
|
| 119 |
+
|
| 120 |
+
void main() {
|
| 121 |
+
const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z);
|
| 122 |
+
|
| 123 |
+
// do NUM_ROWS at a time, unless there aren't enough remaining rows
|
| 124 |
+
if (first_row + NUM_ROWS <= p.stride_d) {
|
| 125 |
+
compute_outputs(first_row, NUM_ROWS);
|
| 126 |
+
} else {
|
| 127 |
+
if (first_row >= p.stride_d) {
|
| 128 |
+
return;
|
| 129 |
+
}
|
| 130 |
+
compute_outputs(first_row, p.stride_d - first_row);
|
| 131 |
+
}
|
| 132 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q4_k.comp
ADDED
|
@@ -0,0 +1,134 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
|
| 3 |
+
#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
|
| 4 |
+
|
| 5 |
+
#include "mul_mat_vec_base.glsl"
|
| 6 |
+
|
| 7 |
+
layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
|
| 8 |
+
|
| 9 |
+
FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];
|
| 10 |
+
|
| 11 |
+
void calc_superblock(const uint a_offset, const uint b_offset, const uint v_im, const uint q_offset, const uint y_offset, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) {
|
| 12 |
+
const uint y1_idx = i * QUANT_K + y_offset;
|
| 13 |
+
const uint y2_idx = y1_idx + 128;
|
| 14 |
+
|
| 15 |
+
[[unroll]] for (uint n = 0; n < num_rows; ++n) {
|
| 16 |
+
const uint ib0 = a_offset + (first_row+n)*num_blocks_per_row;
|
| 17 |
+
const FLOAT_TYPEV2 dm = FLOAT_TYPEV2(data_a[ib0 + i].dm);
|
| 18 |
+
|
| 19 |
+
const uint32_t scale0_u32 = data_a_packed16[ib0 + i].scales[v_im ];
|
| 20 |
+
const uint32_t scale4_u32 = data_a_packed16[ib0 + i].scales[v_im + 2];
|
| 21 |
+
const uint32_t scale8_u32 = data_a_packed16[ib0 + i].scales[v_im + 4];
|
| 22 |
+
|
| 23 |
+
const uint32_t scale_0_4_l = (scale4_u32 << 16) | scale0_u32;
|
| 24 |
+
const uint32_t scale_0_4_h = (scale_0_4_l & 0xC0C0C0C0) >> 2;
|
| 25 |
+
const vec4 scale_0_4_l_f = vec4(unpack8(scale_0_4_l & 0x3F3F3F3F));
|
| 26 |
+
const vec4 scale8_f = vec4(unpack8((((scale8_u32 << 12) | scale8_u32) & 0x0F0F0F0F) | scale_0_4_h));
|
| 27 |
+
|
| 28 |
+
const FLOAT_TYPE sc0 = scale_0_4_l_f.x;
|
| 29 |
+
const FLOAT_TYPE sc1 = scale_0_4_l_f.y;
|
| 30 |
+
const FLOAT_TYPE sc2 = scale_0_4_l_f.z;
|
| 31 |
+
const FLOAT_TYPE sc3 = scale_0_4_l_f.w;
|
| 32 |
+
const FLOAT_TYPE sc4 = scale8_f.x;
|
| 33 |
+
const FLOAT_TYPE sc5 = scale8_f.y;
|
| 34 |
+
const FLOAT_TYPE sc6 = scale8_f.z;
|
| 35 |
+
const FLOAT_TYPE sc7 = scale8_f.w;
|
| 36 |
+
|
| 37 |
+
const uint32_t qs0_u32 = data_a_packed32[ib0 + i].qs[q_offset / 4];
|
| 38 |
+
const uint32_t qs64_u32 = data_a_packed32[ib0 + i].qs[q_offset / 4 + 16];
|
| 39 |
+
|
| 40 |
+
const uint32_t qs0_u32_lo4 = qs0_u32 & 0x0F0F0F0F;
|
| 41 |
+
const uint32_t qs0_u32_hi4 = (qs0_u32 >> 4) & 0x0F0F0F0F;
|
| 42 |
+
const uint32_t qs64_u32_lo4 = qs64_u32 & 0x0F0F0F0F;
|
| 43 |
+
const uint32_t qs64_u32_hi4 = (qs64_u32 >> 4) & 0x0F0F0F0F;
|
| 44 |
+
|
| 45 |
+
const vec4 qs0_lo4 = vec4(unpack8(qs0_u32_lo4));
|
| 46 |
+
const vec4 qs64_lo4 = vec4(unpack8(qs64_u32_lo4));
|
| 47 |
+
const vec4 qs0_hi4 = vec4(unpack8(qs0_u32_hi4));
|
| 48 |
+
const vec4 qs64_hi4 = vec4(unpack8(qs64_u32_hi4));
|
| 49 |
+
|
| 50 |
+
const FLOAT_TYPE q4_0 = qs0_lo4.x;
|
| 51 |
+
const FLOAT_TYPE q4_1 = qs0_lo4.y;
|
| 52 |
+
const FLOAT_TYPE q4_2 = qs0_lo4.z;
|
| 53 |
+
const FLOAT_TYPE q4_3 = qs0_lo4.w;
|
| 54 |
+
const FLOAT_TYPE q4_4 = qs0_hi4.x;
|
| 55 |
+
const FLOAT_TYPE q4_5 = qs0_hi4.y;
|
| 56 |
+
const FLOAT_TYPE q4_6 = qs0_hi4.z;
|
| 57 |
+
const FLOAT_TYPE q4_7 = qs0_hi4.w;
|
| 58 |
+
const FLOAT_TYPE q4_8 = qs64_lo4.x;
|
| 59 |
+
const FLOAT_TYPE q4_9 = qs64_lo4.y;
|
| 60 |
+
const FLOAT_TYPE q4_10 = qs64_lo4.z;
|
| 61 |
+
const FLOAT_TYPE q4_11 = qs64_lo4.w;
|
| 62 |
+
const FLOAT_TYPE q4_12 = qs64_hi4.x;
|
| 63 |
+
const FLOAT_TYPE q4_13 = qs64_hi4.y;
|
| 64 |
+
const FLOAT_TYPE q4_14 = qs64_hi4.z;
|
| 65 |
+
const FLOAT_TYPE q4_15 = qs64_hi4.w;
|
| 66 |
+
|
| 67 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 68 |
+
vec4 by10 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y1_idx) / 4 ]);
|
| 69 |
+
vec4 by132 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y1_idx) / 4 + 8]);
|
| 70 |
+
vec4 by20 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y2_idx) / 4 ]);
|
| 71 |
+
vec4 by232 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y2_idx) / 4 + 8]);
|
| 72 |
+
|
| 73 |
+
const FLOAT_TYPE sx = fma(FLOAT_TYPE(by10.x), q4_0, fma(FLOAT_TYPE(by10.y), q4_1, fma(FLOAT_TYPE(by10.z), q4_2, FLOAT_TYPE(by10.w) * q4_3)));
|
| 74 |
+
const FLOAT_TYPE sy = fma(FLOAT_TYPE(by132.x), q4_4, fma(FLOAT_TYPE(by132.y), q4_5, fma(FLOAT_TYPE(by132.z), q4_6, FLOAT_TYPE(by132.w) * q4_7)));
|
| 75 |
+
const FLOAT_TYPE sz = fma(FLOAT_TYPE(by20.x), q4_8, fma(FLOAT_TYPE(by20.y), q4_9, fma(FLOAT_TYPE(by20.z), q4_10, FLOAT_TYPE(by20.w) * q4_11)));
|
| 76 |
+
const FLOAT_TYPE sw = fma(FLOAT_TYPE(by232.x), q4_12, fma(FLOAT_TYPE(by232.y), q4_13, fma(FLOAT_TYPE(by232.z), q4_14, FLOAT_TYPE(by232.w) * q4_15)));
|
| 77 |
+
const FLOAT_TYPE smin =
|
| 78 |
+
fma(FLOAT_TYPE(by10.x), sc2, fma(FLOAT_TYPE(by132.x), sc3, fma(FLOAT_TYPE(by20.x), sc6, fma(FLOAT_TYPE(by232.x), sc7,
|
| 79 |
+
fma(FLOAT_TYPE(by10.y), sc2, fma(FLOAT_TYPE(by132.y), sc3, fma(FLOAT_TYPE(by20.y), sc6, fma(FLOAT_TYPE(by232.y), sc7,
|
| 80 |
+
fma(FLOAT_TYPE(by10.z), sc2, fma(FLOAT_TYPE(by132.z), sc3, fma(FLOAT_TYPE(by20.z), sc6, fma(FLOAT_TYPE(by232.z), sc7,
|
| 81 |
+
fma(FLOAT_TYPE(by10.w), sc2, fma(FLOAT_TYPE(by132.w), sc3, fma(FLOAT_TYPE(by20.w), sc6, FLOAT_TYPE(by232.w) * sc7)))))))))))))));
|
| 82 |
+
temp[j][n] = fma(dm.x, fma(sx, sc0, fma(sy, sc1, fma(sz, sc4, sw * sc5))), fma(-dm.y, smin, temp[j][n]));
|
| 83 |
+
}
|
| 84 |
+
}
|
| 85 |
+
}
|
| 86 |
+
|
| 87 |
+
void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
|
| 88 |
+
uint a_offset, b_offset, d_offset;
|
| 89 |
+
get_offsets(a_offset, b_offset, d_offset);
|
| 90 |
+
|
| 91 |
+
const uint num_blocks_per_row = p.ncols / QUANT_K;
|
| 92 |
+
|
| 93 |
+
// 16 threads are used to process each block
|
| 94 |
+
const uint it_size = gl_WorkGroupSize.x/16;
|
| 95 |
+
const uint tid = gl_LocalInvocationID.x;
|
| 96 |
+
const uint itid = tid%16; // 0...15
|
| 97 |
+
const uint ix = tid/16;
|
| 98 |
+
|
| 99 |
+
const uint il = itid/4; // 0...3
|
| 100 |
+
const uint ir = itid - 4*il; // 0...3
|
| 101 |
+
const uint n = 4;
|
| 102 |
+
|
| 103 |
+
const uint v_im = il / 2; // 0 or 1. 0 computes 0,32 + 128,160, 1 computes 64,96 + 192,224
|
| 104 |
+
const uint v_in = il % 2;
|
| 105 |
+
|
| 106 |
+
const uint l0 = n * (2 * ir + v_in); // 0...15
|
| 107 |
+
const uint q_offset = 32*v_im + l0;
|
| 108 |
+
const uint y_offset = 64*v_im + l0;
|
| 109 |
+
|
| 110 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 111 |
+
[[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) {
|
| 112 |
+
temp[j][i] = FLOAT_TYPE(0);
|
| 113 |
+
}
|
| 114 |
+
}
|
| 115 |
+
|
| 116 |
+
[[unroll]] for (uint i = ix; i < num_blocks_per_row; i += it_size)
|
| 117 |
+
calc_superblock(a_offset, b_offset, v_im, q_offset, y_offset, i, num_blocks_per_row, first_row, num_rows);
|
| 118 |
+
|
| 119 |
+
reduce_result(temp, d_offset, first_row, num_rows, tid);
|
| 120 |
+
}
|
| 121 |
+
|
| 122 |
+
void main() {
|
| 123 |
+
const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z);
|
| 124 |
+
|
| 125 |
+
// do NUM_ROWS at a time, unless there aren't enough remaining rows
|
| 126 |
+
if (first_row + NUM_ROWS <= p.stride_d) {
|
| 127 |
+
compute_outputs(first_row, NUM_ROWS);
|
| 128 |
+
} else {
|
| 129 |
+
if (first_row >= p.stride_d) {
|
| 130 |
+
return;
|
| 131 |
+
}
|
| 132 |
+
compute_outputs(first_row, p.stride_d - first_row);
|
| 133 |
+
}
|
| 134 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q5_k.comp
ADDED
|
@@ -0,0 +1,165 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
|
| 3 |
+
#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
|
| 4 |
+
|
| 5 |
+
#include "mul_mat_vec_base.glsl"
|
| 6 |
+
|
| 7 |
+
layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
|
| 8 |
+
|
| 9 |
+
FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];
|
| 10 |
+
|
| 11 |
+
void calc_superblock(const uint a_offset, const uint b_offset, const uint v_im, const uint l0, const uint q_offset, const uint y_offset, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows) {
|
| 12 |
+
const uint y1_idx = i * QUANT_K + y_offset;
|
| 13 |
+
const uint y2_idx = y1_idx + 128;
|
| 14 |
+
|
| 15 |
+
[[unroll]] for (uint n = 0; n < num_rows; ++n) {
|
| 16 |
+
const uint ib0 = a_offset + (first_row+n)*num_blocks_per_row;
|
| 17 |
+
const FLOAT_TYPEV2 dm = FLOAT_TYPEV2(data_a[ib0 + i].dm);
|
| 18 |
+
|
| 19 |
+
const uint32_t scale0_u32 = data_a_packed16[ib0 + i].scales[v_im ];
|
| 20 |
+
const uint32_t scale4_u32 = data_a_packed16[ib0 + i].scales[v_im + 2];
|
| 21 |
+
const uint32_t scale8_u32 = data_a_packed16[ib0 + i].scales[v_im + 4];
|
| 22 |
+
|
| 23 |
+
const uint32_t scale_0_4_l = (scale4_u32 << 16) | scale0_u32;
|
| 24 |
+
const uint32_t scale_0_4_h = (scale_0_4_l & 0xC0C0C0C0) >> 2;
|
| 25 |
+
const vec4 scale_0_4_l_f = vec4(unpack8(scale_0_4_l & 0x3F3F3F3F));
|
| 26 |
+
const vec4 scale8_f = vec4(unpack8((((scale8_u32 << 12) | scale8_u32) & 0x0F0F0F0F) | scale_0_4_h));
|
| 27 |
+
|
| 28 |
+
const FLOAT_TYPE sc0 = scale_0_4_l_f.x;
|
| 29 |
+
const FLOAT_TYPE sc1 = scale_0_4_l_f.y;
|
| 30 |
+
const FLOAT_TYPE sc2 = scale_0_4_l_f.z;
|
| 31 |
+
const FLOAT_TYPE sc3 = scale_0_4_l_f.w;
|
| 32 |
+
const FLOAT_TYPE sc4 = scale8_f.x;
|
| 33 |
+
const FLOAT_TYPE sc5 = scale8_f.y;
|
| 34 |
+
const FLOAT_TYPE sc6 = scale8_f.z;
|
| 35 |
+
const FLOAT_TYPE sc7 = scale8_f.w;
|
| 36 |
+
|
| 37 |
+
const uint32_t qs0_16_u32 = uint32_t(data_a_packed16[ib0 + i].qs[q_offset / 2]) | (uint32_t(data_a_packed16[ib0 + i].qs[q_offset / 2 + 8]) << 16);
|
| 38 |
+
const uint32_t qs64_80_u32 = uint32_t(data_a_packed16[ib0 + i].qs[q_offset / 2 + 32]) | (uint32_t(data_a_packed16[ib0 + i].qs[q_offset / 2 + 40]) << 16);
|
| 39 |
+
|
| 40 |
+
uint32_t qs0_16_u32_lo4 = qs0_16_u32 & 0x0F0F0F0F;
|
| 41 |
+
uint32_t qs0_16_u32_hi4 = (qs0_16_u32 >> 4) & 0x0F0F0F0F;
|
| 42 |
+
uint32_t qs64_80_u32_lo4 = qs64_80_u32 & 0x0F0F0F0F;
|
| 43 |
+
uint32_t qs64_80_u32_hi4 = (qs64_80_u32 >> 4) & 0x0F0F0F0F;
|
| 44 |
+
|
| 45 |
+
const uint32_t qh = pack32(u16vec2(data_a_packed16[ib0 + i].qh[l0 / 2], data_a_packed16[ib0 + i].qh[l0 / 2 + 8]));
|
| 46 |
+
|
| 47 |
+
const uint32_t qs0_16_lo4_offset16 = ((qh >> (2*v_im)) & 0x01010101) << 4;
|
| 48 |
+
const uint32_t qs0_16_hi4_offset16 = ((qh >> (2*v_im)) & 0x02020202) << 3;
|
| 49 |
+
const uint32_t qs64_80_lo4_offset16 = ((qh >> (2*v_im)) & 0x10101010);
|
| 50 |
+
const uint32_t qs64_80_hi4_offset16 = ((qh >> (2*v_im)) & 0x20202020) >> 1;
|
| 51 |
+
|
| 52 |
+
qs0_16_u32_lo4 += qs0_16_lo4_offset16;
|
| 53 |
+
qs0_16_u32_hi4 += qs0_16_hi4_offset16;
|
| 54 |
+
qs64_80_u32_lo4 += qs64_80_lo4_offset16;
|
| 55 |
+
qs64_80_u32_hi4 += qs64_80_hi4_offset16;
|
| 56 |
+
|
| 57 |
+
const vec4 qs0_16_lo4 = vec4(unpack8(qs0_16_u32_lo4));
|
| 58 |
+
const vec4 qs64_80_lo4 = vec4(unpack8(qs64_80_u32_lo4));
|
| 59 |
+
const vec4 qs0_16_hi4 = vec4(unpack8(qs0_16_u32_hi4));
|
| 60 |
+
const vec4 qs64_80_hi4 = vec4(unpack8(qs64_80_u32_hi4));
|
| 61 |
+
|
| 62 |
+
const FLOAT_TYPE q4_0 = qs0_16_lo4.x;
|
| 63 |
+
const FLOAT_TYPE q4_1 = qs0_16_lo4.y;
|
| 64 |
+
const FLOAT_TYPE q4_2 = qs0_16_lo4.z;
|
| 65 |
+
const FLOAT_TYPE q4_3 = qs0_16_lo4.w;
|
| 66 |
+
const FLOAT_TYPE q4_4 = qs0_16_hi4.x;
|
| 67 |
+
const FLOAT_TYPE q4_5 = qs0_16_hi4.y;
|
| 68 |
+
const FLOAT_TYPE q4_6 = qs0_16_hi4.z;
|
| 69 |
+
const FLOAT_TYPE q4_7 = qs0_16_hi4.w;
|
| 70 |
+
const FLOAT_TYPE q4_8 = qs64_80_lo4.x;
|
| 71 |
+
const FLOAT_TYPE q4_9 = qs64_80_lo4.y;
|
| 72 |
+
const FLOAT_TYPE q4_10 = qs64_80_lo4.z;
|
| 73 |
+
const FLOAT_TYPE q4_11 = qs64_80_lo4.w;
|
| 74 |
+
const FLOAT_TYPE q4_12 = qs64_80_hi4.x;
|
| 75 |
+
const FLOAT_TYPE q4_13 = qs64_80_hi4.y;
|
| 76 |
+
const FLOAT_TYPE q4_14 = qs64_80_hi4.z;
|
| 77 |
+
const FLOAT_TYPE q4_15 = qs64_80_hi4.w;
|
| 78 |
+
|
| 79 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 80 |
+
vec2 by10 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y1_idx) / 2 ]);
|
| 81 |
+
vec2 by116 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y1_idx) / 2 + 8]);
|
| 82 |
+
vec2 by132 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y1_idx) / 2 + 16]);
|
| 83 |
+
vec2 by148 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y1_idx) / 2 + 24]);
|
| 84 |
+
vec2 by20 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y2_idx) / 2 ]);
|
| 85 |
+
vec2 by216 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y2_idx) / 2 + 8]);
|
| 86 |
+
vec2 by232 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y2_idx) / 2 + 16]);
|
| 87 |
+
vec2 by248 = vec2(data_b_v2[(j*p.batch_stride_b + b_offset + y2_idx) / 2 + 24]);
|
| 88 |
+
|
| 89 |
+
const FLOAT_TYPE sx =
|
| 90 |
+
fma(FLOAT_TYPE(by10.x), q4_0,
|
| 91 |
+
fma(FLOAT_TYPE(by10.y), q4_1,
|
| 92 |
+
fma(FLOAT_TYPE(by116.x), q4_2,
|
| 93 |
+
FLOAT_TYPE(by116.y) * q4_3)));
|
| 94 |
+
const FLOAT_TYPE sy =
|
| 95 |
+
fma(FLOAT_TYPE(by132.x), q4_4,
|
| 96 |
+
fma(FLOAT_TYPE(by132.y), q4_5,
|
| 97 |
+
fma(FLOAT_TYPE(by148.x), q4_6,
|
| 98 |
+
FLOAT_TYPE(by148.y) * q4_7)));
|
| 99 |
+
const FLOAT_TYPE sz =
|
| 100 |
+
fma(FLOAT_TYPE(by20.x), q4_8,
|
| 101 |
+
fma(FLOAT_TYPE(by20.y), q4_9,
|
| 102 |
+
fma(FLOAT_TYPE(by216.x), q4_10,
|
| 103 |
+
FLOAT_TYPE(by216.y) * q4_11)));
|
| 104 |
+
const FLOAT_TYPE sw =
|
| 105 |
+
fma(FLOAT_TYPE(by232.x), q4_12,
|
| 106 |
+
fma(FLOAT_TYPE(by232.y), q4_13,
|
| 107 |
+
fma(FLOAT_TYPE(by248.x), q4_14,
|
| 108 |
+
FLOAT_TYPE(by248.y) * q4_15)));
|
| 109 |
+
const FLOAT_TYPE smin =
|
| 110 |
+
fma(FLOAT_TYPE(by10.x) + FLOAT_TYPE(by10.y) + FLOAT_TYPE(by116.x) + FLOAT_TYPE(by116.y), sc2,
|
| 111 |
+
fma(FLOAT_TYPE(by132.x) + FLOAT_TYPE(by132.y) + FLOAT_TYPE(by148.x) + FLOAT_TYPE(by148.y), sc3,
|
| 112 |
+
fma(FLOAT_TYPE(by20.x) + FLOAT_TYPE(by20.y) + FLOAT_TYPE(by216.x) + FLOAT_TYPE(by216.y), sc6,
|
| 113 |
+
(FLOAT_TYPE(by232.x) + FLOAT_TYPE(by232.y) + FLOAT_TYPE(by248.x) + FLOAT_TYPE(by248.y)) * sc7)));
|
| 114 |
+
temp[j][n] = fma(dm.x, fma(sx, sc0, fma(sy, sc1, fma(sz, sc4, sw * sc5))), fma(-dm.y, smin, temp[j][n]));
|
| 115 |
+
}
|
| 116 |
+
}
|
| 117 |
+
}
|
| 118 |
+
|
| 119 |
+
void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
|
| 120 |
+
uint a_offset, b_offset, d_offset;
|
| 121 |
+
get_offsets(a_offset, b_offset, d_offset);
|
| 122 |
+
|
| 123 |
+
const uint num_blocks_per_row = p.ncols / QUANT_K;
|
| 124 |
+
|
| 125 |
+
// 16 threads are used to process each block
|
| 126 |
+
const uint it_size = gl_WorkGroupSize.x/16;
|
| 127 |
+
const uint tid = gl_LocalInvocationID.x;
|
| 128 |
+
const uint itid = tid%16; // 0...15
|
| 129 |
+
const uint ix = tid/16;
|
| 130 |
+
|
| 131 |
+
const uint il = itid/4; // 0...3
|
| 132 |
+
const uint ir = itid - 4*il; // 0...3
|
| 133 |
+
|
| 134 |
+
const uint v_im = il / 2; // 0 or 1. 0 computes 0,32 + 128,160, 1 computes 64,96 + 192,224
|
| 135 |
+
const uint v_in = il % 2;
|
| 136 |
+
|
| 137 |
+
const uint l0 = 4*ir + 2*v_in; // 0...15
|
| 138 |
+
const uint q_offset = 32*v_im + l0;
|
| 139 |
+
const uint y_offset = 64*v_im + l0;
|
| 140 |
+
|
| 141 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 142 |
+
[[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) {
|
| 143 |
+
temp[j][i] = FLOAT_TYPE(0);
|
| 144 |
+
}
|
| 145 |
+
}
|
| 146 |
+
|
| 147 |
+
[[unroll]] for (uint i = ix; i < num_blocks_per_row; i += it_size)
|
| 148 |
+
calc_superblock(a_offset, b_offset, v_im, l0, q_offset, y_offset, i, num_blocks_per_row, first_row, num_rows);
|
| 149 |
+
|
| 150 |
+
reduce_result(temp, d_offset, first_row, num_rows, tid);
|
| 151 |
+
}
|
| 152 |
+
|
| 153 |
+
void main() {
|
| 154 |
+
const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z);
|
| 155 |
+
|
| 156 |
+
// do NUM_ROWS at a time, unless there aren't enough remaining rows
|
| 157 |
+
if (first_row + NUM_ROWS <= p.stride_d) {
|
| 158 |
+
compute_outputs(first_row, NUM_ROWS);
|
| 159 |
+
} else {
|
| 160 |
+
if (first_row >= p.stride_d) {
|
| 161 |
+
return;
|
| 162 |
+
}
|
| 163 |
+
compute_outputs(first_row, p.stride_d - first_row);
|
| 164 |
+
}
|
| 165 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vec_q6_k.comp
ADDED
|
@@ -0,0 +1,130 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
|
| 3 |
+
#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
|
| 4 |
+
|
| 5 |
+
#include "mul_mat_vec_base.glsl"
|
| 6 |
+
|
| 7 |
+
layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
|
| 8 |
+
|
| 9 |
+
shared FLOAT_TYPE sccache[2][BLOCK_SIZE/16][16];
|
| 10 |
+
|
| 11 |
+
FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];
|
| 12 |
+
uint csel = 0;
|
| 13 |
+
|
| 14 |
+
void calc_superblock(const uint a_offset, const uint b_offset, const uint itid, const uint ix, const uint ql_offset, const uint qh_offset, const uint s_offset, const uint y_offset, const uint i, const uint num_blocks_per_row, const uint first_row, const uint num_rows, const bool all_threads) {
|
| 15 |
+
const uint y_idx = i * QUANT_K + y_offset;
|
| 16 |
+
|
| 17 |
+
[[unroll]] for (uint n = 0; n < num_rows; ++n) {
|
| 18 |
+
const uint ib0 = a_offset + (first_row+n)*num_blocks_per_row;
|
| 19 |
+
csel ^= 1;
|
| 20 |
+
|
| 21 |
+
if (!all_threads) { // when we don't have enough blocks to use all threads
|
| 22 |
+
if (i < num_blocks_per_row)
|
| 23 |
+
sccache[csel][ix][itid] = FLOAT_TYPE(data_a[ib0 + i].scales[itid]);
|
| 24 |
+
barrier();
|
| 25 |
+
|
| 26 |
+
if (i >= num_blocks_per_row)
|
| 27 |
+
continue;
|
| 28 |
+
}
|
| 29 |
+
|
| 30 |
+
const uint32_t ql0_u32 = uint32_t(data_a_packed16[ib0 + i].ql[ql_offset / 2]) | (uint32_t(data_a_packed16[ib0 + i].ql[ql_offset / 2 + 1]) << 16);
|
| 31 |
+
const uint32_t ql32_u32 = uint32_t(data_a_packed16[ib0 + i].ql[ql_offset / 2 + 16]) | (uint32_t(data_a_packed16[ib0 + i].ql[ql_offset / 2 + 17]) << 16);
|
| 32 |
+
|
| 33 |
+
const uint32_t ql0_u32_lo4 = ql0_u32 & 0x0F0F0F0F;
|
| 34 |
+
const uint32_t ql0_u32_hi4 = (ql0_u32 >> 4) & 0x0F0F0F0F;
|
| 35 |
+
const uint32_t ql32_u32_lo4 = ql32_u32 & 0x0F0F0F0F;
|
| 36 |
+
const uint32_t ql32_u32_hi4 = (ql32_u32 >> 4) & 0x0F0F0F0F;
|
| 37 |
+
|
| 38 |
+
const uint32_t qh_u32 = uint32_t(data_a_packed16[ib0 + i].qh[qh_offset / 2]) | (uint32_t(data_a_packed16[ib0 + i].qh[qh_offset / 2 + 1]) << 16);
|
| 39 |
+
const uint32_t qh0_u32 = (qh_u32 & 0x03030303) << 4;
|
| 40 |
+
const uint32_t qh2_u32 = (qh_u32 & 0x0C0C0C0C) << 2;
|
| 41 |
+
const uint32_t qh4_u32 = (qh_u32 & 0x30303030);
|
| 42 |
+
const uint32_t qh6_u32 = (qh_u32 & 0xC0C0C0C0) >> 2;
|
| 43 |
+
|
| 44 |
+
const uint32_t q0_u32 = ql0_u32_lo4 | qh0_u32;
|
| 45 |
+
const uint32_t q1_u32 = ql32_u32_lo4 | qh2_u32;
|
| 46 |
+
const uint32_t q2_u32 = ql0_u32_hi4 | qh4_u32;
|
| 47 |
+
const uint32_t q3_u32 = ql32_u32_hi4 | qh6_u32;
|
| 48 |
+
|
| 49 |
+
const vec4 q0 = vec4(unpack8(q0_u32)) - 32;
|
| 50 |
+
const vec4 q1 = vec4(unpack8(q1_u32)) - 32;
|
| 51 |
+
const vec4 q2 = vec4(unpack8(q2_u32)) - 32;
|
| 52 |
+
const vec4 q3 = vec4(unpack8(q3_u32)) - 32;
|
| 53 |
+
|
| 54 |
+
if (all_threads) {
|
| 55 |
+
sccache[csel][ix][itid] = FLOAT_TYPE(data_a[ib0 + i].scales[itid]);
|
| 56 |
+
barrier();
|
| 57 |
+
}
|
| 58 |
+
|
| 59 |
+
const FLOAT_TYPE d = FLOAT_TYPE(data_a[ib0 + i].d);
|
| 60 |
+
|
| 61 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 62 |
+
vec4 by0 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 ]);
|
| 63 |
+
vec4 by32 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 8]);
|
| 64 |
+
vec4 by64 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 16]);
|
| 65 |
+
vec4 by96 = vec4(data_b_v4[(j*p.batch_stride_b + b_offset + y_idx) / 4 + 24]);
|
| 66 |
+
|
| 67 |
+
FLOAT_TYPE sum[4] = {0, 0, 0, 0};
|
| 68 |
+
[[unroll]] for (uint l = 0; l < 4; ++l) {
|
| 69 |
+
sum[0] = fma(FLOAT_TYPE(by0[l]), q0[l], sum[0]);
|
| 70 |
+
sum[1] = fma(FLOAT_TYPE(by32[l]), q1[l], sum[1]);
|
| 71 |
+
sum[2] = fma(FLOAT_TYPE(by64[l]), q2[l], sum[2]);
|
| 72 |
+
sum[3] = fma(FLOAT_TYPE(by96[l]), q3[l], sum[3]);
|
| 73 |
+
}
|
| 74 |
+
temp[j][n] = fma(fma(sum[0], sccache[csel][ix][s_offset], fma(sum[1], sccache[csel][ix][s_offset + 2], fma(sum[2], sccache[csel][ix][s_offset + 4], sum[3] * sccache[csel][ix][s_offset + 6]))), d, temp[j][n]);
|
| 75 |
+
}
|
| 76 |
+
}
|
| 77 |
+
}
|
| 78 |
+
|
| 79 |
+
void compute_outputs(const uint first_row, const uint num_rows) {
|
| 80 |
+
uint a_offset, b_offset, d_offset;
|
| 81 |
+
get_offsets(a_offset, b_offset, d_offset);
|
| 82 |
+
|
| 83 |
+
const uint num_blocks_per_row = p.ncols / QUANT_K;
|
| 84 |
+
|
| 85 |
+
// 16 threads are used to process each block
|
| 86 |
+
const uint it_size = gl_WorkGroupSize.x/16;
|
| 87 |
+
const uint tid = gl_LocalInvocationID.x;
|
| 88 |
+
const uint itid = tid%16; // 0...15
|
| 89 |
+
const uint ix = tid/16;
|
| 90 |
+
|
| 91 |
+
const uint v_im = itid/8; // 0 or 1. 0 computes 0..., 1 computes 128...
|
| 92 |
+
const uint v_in = itid - 8*v_im; // 0...7
|
| 93 |
+
|
| 94 |
+
const uint l0 = 4 * v_in; // 0, 4, 8, ..., 28
|
| 95 |
+
const uint is = v_in / 4;
|
| 96 |
+
|
| 97 |
+
const uint ql_offset = 64*v_im + l0;
|
| 98 |
+
const uint qh_offset = 32*v_im + l0;
|
| 99 |
+
const uint s_offset = 8*v_im + is;
|
| 100 |
+
const uint y_offset = 128*v_im + l0;
|
| 101 |
+
|
| 102 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 103 |
+
[[unroll]] for (uint i = 0; i < NUM_ROWS; ++i) {
|
| 104 |
+
temp[j][i] = FLOAT_TYPE(0);
|
| 105 |
+
}
|
| 106 |
+
}
|
| 107 |
+
|
| 108 |
+
const uint nbr_par_th = num_blocks_per_row%it_size;
|
| 109 |
+
const uint nbr_all_th = num_blocks_per_row - nbr_par_th;
|
| 110 |
+
uint i0 = 0;
|
| 111 |
+
[[unroll]] for (; i0 < nbr_all_th; i0 += it_size)
|
| 112 |
+
calc_superblock(a_offset, b_offset, itid, ix, ql_offset, qh_offset, s_offset, y_offset, i0 + ix, num_blocks_per_row, first_row, num_rows, true);
|
| 113 |
+
calc_superblock(a_offset, b_offset, itid, ix, ql_offset, qh_offset, s_offset, y_offset, i0 + ix, num_blocks_per_row, first_row, num_rows, false);
|
| 114 |
+
|
| 115 |
+
reduce_result(temp, d_offset, first_row, num_rows, tid);
|
| 116 |
+
}
|
| 117 |
+
|
| 118 |
+
void main() {
|
| 119 |
+
const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z);
|
| 120 |
+
|
| 121 |
+
// do NUM_ROWS at a time, unless there aren't enough remaining rows
|
| 122 |
+
if (first_row + NUM_ROWS <= p.stride_d) {
|
| 123 |
+
compute_outputs(first_row, NUM_ROWS);
|
| 124 |
+
} else {
|
| 125 |
+
if (first_row >= p.stride_d) {
|
| 126 |
+
return;
|
| 127 |
+
}
|
| 128 |
+
compute_outputs(first_row, p.stride_d - first_row);
|
| 129 |
+
}
|
| 130 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq.comp
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
|
| 3 |
+
#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
|
| 4 |
+
#extension GL_EXT_integer_dot_product : require
|
| 5 |
+
|
| 6 |
+
#define MMQ
|
| 7 |
+
#define NEEDS_IQ1S_GRID_GPU
|
| 8 |
+
#define B_TYPE block_q8_1_x4
|
| 9 |
+
|
| 10 |
+
#include "mul_mat_vec_base.glsl"
|
| 11 |
+
|
| 12 |
+
layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
|
| 13 |
+
|
| 14 |
+
#if defined(DATA_A_QUANT_LEGACY) || defined(DATA_A_MXFP4)
|
| 15 |
+
#define K_PER_ITER 8
|
| 16 |
+
#elif defined(DATA_A_QUANT_K)
|
| 17 |
+
#define K_PER_ITER 16
|
| 18 |
+
#elif defined(DATA_A_IQ1_S) || defined(DATA_A_IQ1_M)
|
| 19 |
+
#define K_PER_ITER 32
|
| 20 |
+
#else
|
| 21 |
+
#error unimplemented
|
| 22 |
+
#endif
|
| 23 |
+
|
| 24 |
+
uint a_offset, b_offset, d_offset;
|
| 25 |
+
|
| 26 |
+
int32_t cache_b_qs[K_PER_ITER / 4];
|
| 27 |
+
vec2 cache_b_ds;
|
| 28 |
+
|
| 29 |
+
#include "mul_mat_vecq_funcs.glsl"
|
| 30 |
+
|
| 31 |
+
void iter(inout FLOAT_TYPE temp[NUM_COLS][NUM_ROWS], const uint first_row, const uint num_rows, const uint col, const uint b_qs_idx) {
|
| 32 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 33 |
+
// Preload data_b block
|
| 34 |
+
const uint b_block_idx = (j*p.batch_stride_b + col) / QUANT_K_Q8_1 + b_offset;
|
| 35 |
+
const uint b_block_idx_outer = b_block_idx / 4;
|
| 36 |
+
const uint b_block_idx_inner = b_block_idx % 4;
|
| 37 |
+
cache_b_ds = vec2(data_b[b_block_idx_outer].ds[b_block_idx_inner]);
|
| 38 |
+
|
| 39 |
+
#if QUANT_R == 2
|
| 40 |
+
// Assumes K_PER_ITER == 8
|
| 41 |
+
cache_b_qs[0] = data_b[b_block_idx_outer].qs[b_block_idx_inner * 8 + b_qs_idx];
|
| 42 |
+
cache_b_qs[1] = data_b[b_block_idx_outer].qs[b_block_idx_inner * 8 + b_qs_idx + 4];
|
| 43 |
+
#else
|
| 44 |
+
#if K_PER_ITER == 8
|
| 45 |
+
cache_b_qs[0] = data_b[b_block_idx_outer].qs[b_block_idx_inner * 8 + b_qs_idx * 2];
|
| 46 |
+
cache_b_qs[1] = data_b[b_block_idx_outer].qs[b_block_idx_inner * 8 + b_qs_idx * 2 + 1];
|
| 47 |
+
#elif K_PER_ITER == 16
|
| 48 |
+
cache_b_qs[0] = data_b[b_block_idx_outer].qs[b_block_idx_inner * 8 + b_qs_idx * 4 ];
|
| 49 |
+
cache_b_qs[1] = data_b[b_block_idx_outer].qs[b_block_idx_inner * 8 + b_qs_idx * 4 + 1];
|
| 50 |
+
cache_b_qs[2] = data_b[b_block_idx_outer].qs[b_block_idx_inner * 8 + b_qs_idx * 4 + 2];
|
| 51 |
+
cache_b_qs[3] = data_b[b_block_idx_outer].qs[b_block_idx_inner * 8 + b_qs_idx * 4 + 3];
|
| 52 |
+
#elif K_PER_ITER == 32
|
| 53 |
+
cache_b_qs[0] = data_b[b_block_idx_outer].qs[b_block_idx_inner * 8 ];
|
| 54 |
+
cache_b_qs[1] = data_b[b_block_idx_outer].qs[b_block_idx_inner * 8 + 1];
|
| 55 |
+
cache_b_qs[2] = data_b[b_block_idx_outer].qs[b_block_idx_inner * 8 + 2];
|
| 56 |
+
cache_b_qs[3] = data_b[b_block_idx_outer].qs[b_block_idx_inner * 8 + 3];
|
| 57 |
+
cache_b_qs[4] = data_b[b_block_idx_outer].qs[b_block_idx_inner * 8 + 4];
|
| 58 |
+
cache_b_qs[5] = data_b[b_block_idx_outer].qs[b_block_idx_inner * 8 + 5];
|
| 59 |
+
cache_b_qs[6] = data_b[b_block_idx_outer].qs[b_block_idx_inner * 8 + 6];
|
| 60 |
+
cache_b_qs[7] = data_b[b_block_idx_outer].qs[b_block_idx_inner * 8 + 7];
|
| 61 |
+
#else
|
| 62 |
+
#error unimplemented
|
| 63 |
+
#endif
|
| 64 |
+
#endif
|
| 65 |
+
|
| 66 |
+
uint ibi = first_row*p.ncols;
|
| 67 |
+
[[unroll]] for (uint n = 0; n < num_rows; ++n) {
|
| 68 |
+
const uint a_block_idx = (ibi + col)/QUANT_K_Q8_1 + a_offset;
|
| 69 |
+
ibi += p.ncols;
|
| 70 |
+
|
| 71 |
+
temp[j][n] += mmvq_dot_product(a_block_idx, b_qs_idx);
|
| 72 |
+
}
|
| 73 |
+
}
|
| 74 |
+
}
|
| 75 |
+
|
| 76 |
+
void compute_outputs(const uint32_t first_row, const uint32_t num_rows) {
|
| 77 |
+
const uint tid = gl_LocalInvocationID.x;
|
| 78 |
+
|
| 79 |
+
get_offsets(a_offset, b_offset, d_offset);
|
| 80 |
+
a_offset *= QUANT_K / QUANT_K_Q8_1;
|
| 81 |
+
b_offset /= QUANT_K_Q8_1;
|
| 82 |
+
|
| 83 |
+
FLOAT_TYPE temp[NUM_COLS][NUM_ROWS];
|
| 84 |
+
|
| 85 |
+
[[unroll]] for (uint j = 0; j < NUM_COLS; ++j) {
|
| 86 |
+
[[unroll]] for (uint n = 0; n < num_rows; ++n) {
|
| 87 |
+
temp[j][n] = FLOAT_TYPE(0.0f);
|
| 88 |
+
}
|
| 89 |
+
}
|
| 90 |
+
|
| 91 |
+
const uint col_stride = K_PER_ITER * BLOCK_SIZE;
|
| 92 |
+
uint num_iters = p.ncols / col_stride;
|
| 93 |
+
if (num_iters * col_stride + K_PER_ITER * tid < p.ncols) {
|
| 94 |
+
num_iters++;
|
| 95 |
+
}
|
| 96 |
+
|
| 97 |
+
const uint b_qs_idx = tid % (32 / K_PER_ITER);
|
| 98 |
+
uint col = tid * K_PER_ITER;
|
| 99 |
+
while (num_iters >= 4) {
|
| 100 |
+
// Manually partially unroll the loop
|
| 101 |
+
[[unroll]] for (uint k = 0; k < 4; ++k) {
|
| 102 |
+
iter(temp, first_row, num_rows, col, b_qs_idx);
|
| 103 |
+
col += col_stride;
|
| 104 |
+
}
|
| 105 |
+
|
| 106 |
+
num_iters -= 4;
|
| 107 |
+
}
|
| 108 |
+
|
| 109 |
+
if (num_iters >= 2) {
|
| 110 |
+
// Manually partially unroll the loop
|
| 111 |
+
iter(temp, first_row, num_rows, col, b_qs_idx);
|
| 112 |
+
col += col_stride;
|
| 113 |
+
iter(temp, first_row, num_rows, col, b_qs_idx);
|
| 114 |
+
col += col_stride;
|
| 115 |
+
num_iters -= 2;
|
| 116 |
+
}
|
| 117 |
+
|
| 118 |
+
if (num_iters > 0) {
|
| 119 |
+
iter(temp, first_row, num_rows, col, b_qs_idx);
|
| 120 |
+
}
|
| 121 |
+
|
| 122 |
+
reduce_result(temp, d_offset, first_row, num_rows, tid);
|
| 123 |
+
}
|
| 124 |
+
|
| 125 |
+
void main() {
|
| 126 |
+
const uint first_row = NUM_ROWS * (gl_WorkGroupID.x + gl_NumWorkGroups.x * gl_WorkGroupID.z);
|
| 127 |
+
|
| 128 |
+
#ifdef NEEDS_INIT_IQ_SHMEM
|
| 129 |
+
init_iq_shmem(gl_WorkGroupSize);
|
| 130 |
+
#endif
|
| 131 |
+
|
| 132 |
+
// do NUM_ROWS at a time, unless there aren't enough remaining rows
|
| 133 |
+
if (first_row + NUM_ROWS <= p.stride_d) {
|
| 134 |
+
compute_outputs(first_row, NUM_ROWS);
|
| 135 |
+
} else {
|
| 136 |
+
if (first_row >= p.stride_d) {
|
| 137 |
+
return;
|
| 138 |
+
}
|
| 139 |
+
compute_outputs(first_row, p.stride_d - first_row);
|
| 140 |
+
}
|
| 141 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mat_vecq_funcs.glsl
ADDED
|
@@ -0,0 +1,527 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
|
| 2 |
+
#extension GL_EXT_shader_explicit_arithmetic_types_int16 : require
|
| 3 |
+
#extension GL_EXT_shader_explicit_arithmetic_types_int8 : require
|
| 4 |
+
|
| 5 |
+
#include "types.glsl"
|
| 6 |
+
|
| 7 |
+
#if defined(DATA_A_Q4_0) || defined(DATA_A_Q5_0) || defined(DATA_A_Q8_0) || defined(DATA_A_IQ1_S) || defined(DATA_A_IQ2_XXS) || defined(DATA_A_IQ2_XS) || defined(DATA_A_IQ2_S) || defined(DATA_A_IQ3_XXS) || defined(DATA_A_IQ3_S) || defined(DATA_A_IQ4_XS) || defined(DATA_A_IQ4_NL)
|
| 8 |
+
FLOAT_TYPE get_dm(uint ib) {
|
| 9 |
+
return FLOAT_TYPE(data_a[ib].d);
|
| 10 |
+
}
|
| 11 |
+
#endif
|
| 12 |
+
|
| 13 |
+
#if defined(DATA_A_Q4_1) || defined(DATA_A_Q5_1)
|
| 14 |
+
FLOAT_TYPEV2 get_dm(uint ib) {
|
| 15 |
+
return FLOAT_TYPEV2(data_a_packed32[ib].dm);
|
| 16 |
+
}
|
| 17 |
+
#endif
|
| 18 |
+
|
| 19 |
+
#if defined(DATA_A_MXFP4)
|
| 20 |
+
FLOAT_TYPE get_dm(uint ib) {
|
| 21 |
+
return FLOAT_TYPE(e8m0_to_fp32(data_a[ib].e));
|
| 22 |
+
}
|
| 23 |
+
#endif
|
| 24 |
+
|
| 25 |
+
#if defined(DATA_A_Q2_K)
|
| 26 |
+
FLOAT_TYPEV2 get_dm(uint ib) {
|
| 27 |
+
const uint ib_k = ib / 8;
|
| 28 |
+
return FLOAT_TYPEV2(data_a_packed32[ib_k].dm);
|
| 29 |
+
}
|
| 30 |
+
#endif
|
| 31 |
+
|
| 32 |
+
// Each iqs value maps to a 32-bit integer
|
| 33 |
+
#if defined(DATA_A_Q4_0)
|
| 34 |
+
// 2-byte loads for Q4_0 blocks (18 bytes)
|
| 35 |
+
i32vec2 repack(uint ib, uint iqs) {
|
| 36 |
+
const u16vec2 quants = u16vec2(data_a_packed16[ib].qs[iqs * 2 ],
|
| 37 |
+
data_a_packed16[ib].qs[iqs * 2 + 1]);
|
| 38 |
+
const uint32_t vui = pack32(quants);
|
| 39 |
+
return i32vec2( vui & 0x0F0F0F0F,
|
| 40 |
+
(vui >> 4) & 0x0F0F0F0F);
|
| 41 |
+
}
|
| 42 |
+
|
| 43 |
+
FLOAT_TYPE mul_q8_1(const int32_t q_sum, const float da, const vec2 dsb, const int32_t sum_divisor) {
|
| 44 |
+
return FLOAT_TYPE(da * (float(q_sum) * dsb.x - (8 / sum_divisor) * dsb.y));
|
| 45 |
+
}
|
| 46 |
+
#endif
|
| 47 |
+
|
| 48 |
+
#if defined(DATA_A_Q4_1)
|
| 49 |
+
// 4-byte loads for Q4_1 blocks (20 bytes)
|
| 50 |
+
i32vec2 repack(uint ib, uint iqs) {
|
| 51 |
+
const uint32_t vui = data_a_packed32[ib].qs[iqs];
|
| 52 |
+
return i32vec2( vui & 0x0F0F0F0F,
|
| 53 |
+
(vui >> 4) & 0x0F0F0F0F);
|
| 54 |
+
}
|
| 55 |
+
|
| 56 |
+
FLOAT_TYPE mul_q8_1(const int32_t q_sum, const vec2 dma, const vec2 dsb, const int32_t sum_divisor) {
|
| 57 |
+
return FLOAT_TYPE(float(q_sum) * dma.x * dsb.x + dma.y * dsb.y / sum_divisor);
|
| 58 |
+
}
|
| 59 |
+
#endif
|
| 60 |
+
|
| 61 |
+
#if defined(DATA_A_Q5_0)
|
| 62 |
+
// 2-byte loads for Q5_0 blocks (22 bytes)
|
| 63 |
+
i32vec2 repack(uint ib, uint iqs) {
|
| 64 |
+
const u16vec2 quants = u16vec2(data_a_packed16[ib].qs[iqs * 2 ],
|
| 65 |
+
data_a_packed16[ib].qs[iqs * 2 + 1]);
|
| 66 |
+
const uint32_t vui = pack32(quants);
|
| 67 |
+
const int32_t qh = int32_t((uint32_t(data_a_packed16[ib].qh[1]) << 16 | data_a_packed16[ib].qh[0]) >> (4 * iqs));
|
| 68 |
+
const int32_t v0 = int32_t(vui & 0x0F0F0F0F)
|
| 69 |
+
| ((qh & 0xF) * 0x02040810) & 0x10101010; // (0,1,2,3) -> (4,12,20,28)
|
| 70 |
+
|
| 71 |
+
const int32_t v1 = int32_t((vui >> 4) & 0x0F0F0F0F)
|
| 72 |
+
| (((qh >> 16) & 0xF) * 0x02040810) & 0x10101010; // (16,17,18,19) -> (4,12,20,28)
|
| 73 |
+
|
| 74 |
+
return i32vec2(v0, v1);
|
| 75 |
+
}
|
| 76 |
+
|
| 77 |
+
FLOAT_TYPE mul_q8_1(const int32_t q_sum, const float da, const vec2 dsb, const int32_t sum_divisor) {
|
| 78 |
+
return FLOAT_TYPE(da * (float(q_sum) * dsb.x - (16 / sum_divisor) * dsb.y));
|
| 79 |
+
}
|
| 80 |
+
#endif
|
| 81 |
+
|
| 82 |
+
#if defined(DATA_A_Q5_1)
|
| 83 |
+
// 4-byte loads for Q5_1 blocks (24 bytes)
|
| 84 |
+
i32vec2 repack(uint ib, uint iqs) {
|
| 85 |
+
const u16vec2 quants = u16vec2(data_a_packed16[ib].qs[iqs * 2 ],
|
| 86 |
+
data_a_packed16[ib].qs[iqs * 2 + 1]);
|
| 87 |
+
const uint32_t vui = pack32(quants);
|
| 88 |
+
const int32_t qh = int32_t(data_a_packed32[ib].qh >> (4 * iqs));
|
| 89 |
+
const int32_t v0 = int32_t(vui & 0x0F0F0F0F)
|
| 90 |
+
| ((qh & 0xF) * 0x02040810) & 0x10101010; // (0,1,2,3) -> (4,12,20,28)
|
| 91 |
+
|
| 92 |
+
const int32_t v1 = int32_t((vui >> 4) & 0x0F0F0F0F)
|
| 93 |
+
| (((qh >> 16) & 0xF) * 0x02040810) & 0x10101010; // (16,17,18,19) -> (4,12,20,28)
|
| 94 |
+
|
| 95 |
+
return i32vec2(v0, v1);
|
| 96 |
+
}
|
| 97 |
+
|
| 98 |
+
FLOAT_TYPE mul_q8_1(const int32_t q_sum, const vec2 dma, const vec2 dsb, const int32_t sum_divisor) {
|
| 99 |
+
return FLOAT_TYPE(float(q_sum) * dma.x * dsb.x + dma.y * dsb.y / sum_divisor);
|
| 100 |
+
}
|
| 101 |
+
#endif
|
| 102 |
+
|
| 103 |
+
#if defined(DATA_A_Q8_0)
|
| 104 |
+
// 2-byte loads for Q8_0 blocks (34 bytes)
|
| 105 |
+
int32_t repack(uint ib, uint iqs) {
|
| 106 |
+
return pack32(i16vec2(data_a_packed16[ib].qs[iqs * 2 ],
|
| 107 |
+
data_a_packed16[ib].qs[iqs * 2 + 1]));
|
| 108 |
+
}
|
| 109 |
+
|
| 110 |
+
FLOAT_TYPE mul_q8_1(const int32_t q_sum, const float da, const vec2 dsb, const int32_t sum_divisor) {
|
| 111 |
+
return FLOAT_TYPE(float(q_sum) * da * dsb.x);
|
| 112 |
+
}
|
| 113 |
+
#endif
|
| 114 |
+
|
| 115 |
+
#if defined(DATA_A_MXFP4)
|
| 116 |
+
// 1-byte loads for mxfp4 blocks (17 bytes)
|
| 117 |
+
i32vec2 repack(uint ib, uint iqs) {
|
| 118 |
+
const uint32_t qs = pack32(u8vec4(data_a[ib].qs[iqs * 4 ],
|
| 119 |
+
data_a[ib].qs[iqs * 4 + 1],
|
| 120 |
+
data_a[ib].qs[iqs * 4 + 2],
|
| 121 |
+
data_a[ib].qs[iqs * 4 + 3]));
|
| 122 |
+
|
| 123 |
+
const u8vec4 i_a0 = unpack8( qs & 0x0F0F0F0F);
|
| 124 |
+
const u8vec4 i_a1 = unpack8((qs >> 4) & 0x0F0F0F0F);
|
| 125 |
+
|
| 126 |
+
return i32vec2(pack32(i8vec4(kvalues_mxfp4[i_a0.x], kvalues_mxfp4[i_a0.y], kvalues_mxfp4[i_a0.z], kvalues_mxfp4[i_a0.w])),
|
| 127 |
+
pack32(i8vec4(kvalues_mxfp4[i_a1.x], kvalues_mxfp4[i_a1.y], kvalues_mxfp4[i_a1.z], kvalues_mxfp4[i_a1.w])));
|
| 128 |
+
}
|
| 129 |
+
|
| 130 |
+
FLOAT_TYPE mul_q8_1(const int32_t q_sum, const float da, const vec2 dsb, const int32_t sum_divisor) {
|
| 131 |
+
return FLOAT_TYPE(da * dsb.x * float(q_sum) * 0.5);
|
| 132 |
+
}
|
| 133 |
+
#endif
|
| 134 |
+
|
| 135 |
+
#if defined(DATA_A_QUANT_LEGACY) || defined(DATA_A_MXFP4)
|
| 136 |
+
FLOAT_TYPE mmvq_dot_product(const uint ib_a, const uint iqs) {
|
| 137 |
+
int32_t q_sum = 0;
|
| 138 |
+
#if QUANT_R == 2
|
| 139 |
+
const i32vec2 data_a_qs = repack(ib_a, iqs);
|
| 140 |
+
q_sum += dotPacked4x8EXT(data_a_qs.x,
|
| 141 |
+
cache_b_qs[0]);
|
| 142 |
+
q_sum += dotPacked4x8EXT(data_a_qs.y,
|
| 143 |
+
cache_b_qs[1]);
|
| 144 |
+
#else
|
| 145 |
+
int32_t data_a_qs = repack(ib_a, iqs * 2);
|
| 146 |
+
q_sum += dotPacked4x8EXT(data_a_qs,
|
| 147 |
+
cache_b_qs[0]);
|
| 148 |
+
data_a_qs = repack(ib_a, iqs * 2 + 1);
|
| 149 |
+
q_sum += dotPacked4x8EXT(data_a_qs,
|
| 150 |
+
cache_b_qs[1]);
|
| 151 |
+
#endif
|
| 152 |
+
|
| 153 |
+
// 2 quants per call => divide sums by 8/2 = 4
|
| 154 |
+
return mul_q8_1(q_sum, get_dm(ib_a), cache_b_ds, 4);
|
| 155 |
+
}
|
| 156 |
+
#endif
|
| 157 |
+
|
| 158 |
+
#if defined(DATA_A_Q2_K)
|
| 159 |
+
// 4-byte loads for Q2_K blocks (84 bytes)
|
| 160 |
+
i32vec4 repack4(uint ib, uint iqs) {
|
| 161 |
+
const uint ib_k = ib / 8;
|
| 162 |
+
const uint iqs_k = (ib % 8) * 8 + iqs;
|
| 163 |
+
|
| 164 |
+
const uint qs_idx = (iqs_k / 32) * 8 + (iqs_k % 8);
|
| 165 |
+
const uint qs_shift = ((iqs_k % 32) / 8) * 2;
|
| 166 |
+
|
| 167 |
+
return i32vec4((data_a_packed32[ib_k].qs[qs_idx ] >> qs_shift) & 0x03030303,
|
| 168 |
+
(data_a_packed32[ib_k].qs[qs_idx + 1] >> qs_shift) & 0x03030303,
|
| 169 |
+
(data_a_packed32[ib_k].qs[qs_idx + 2] >> qs_shift) & 0x03030303,
|
| 170 |
+
(data_a_packed32[ib_k].qs[qs_idx + 3] >> qs_shift) & 0x03030303);
|
| 171 |
+
}
|
| 172 |
+
|
| 173 |
+
uint8_t get_scale(uint ib, uint iqs) {
|
| 174 |
+
const uint ib_k = ib / 8;
|
| 175 |
+
const uint iqs_k = (ib % 8) * 8 + iqs;
|
| 176 |
+
|
| 177 |
+
return data_a[ib_k].scales[iqs_k / 4];
|
| 178 |
+
}
|
| 179 |
+
|
| 180 |
+
FLOAT_TYPE mmvq_dot_product(const uint ib_a, const uint iqs) {
|
| 181 |
+
int32_t sum_d = 0;
|
| 182 |
+
int32_t sum_m = 0;
|
| 183 |
+
|
| 184 |
+
const i32vec4 qs_a = repack4(ib_a, iqs * 4);
|
| 185 |
+
const uint8_t scale = get_scale(ib_a, iqs * 4);
|
| 186 |
+
const vec2 dm = vec2(get_dm(ib_a));
|
| 187 |
+
const int32_t scale_m = int32_t(scale >> 4) * 0x01010101; // Duplicate 8-bit value across 32-bits.
|
| 188 |
+
|
| 189 |
+
sum_d += dotPacked4x8EXT(qs_a.x, cache_b_qs[0]) * (scale & 0xF);
|
| 190 |
+
sum_m += dotPacked4x8EXT(scale_m, cache_b_qs[0]);
|
| 191 |
+
|
| 192 |
+
sum_d += dotPacked4x8EXT(qs_a.y, cache_b_qs[1]) * (scale & 0xF);
|
| 193 |
+
sum_m += dotPacked4x8EXT(scale_m, cache_b_qs[1]);
|
| 194 |
+
|
| 195 |
+
sum_d += dotPacked4x8EXT(qs_a.z, cache_b_qs[2]) * (scale & 0xF);
|
| 196 |
+
sum_m += dotPacked4x8EXT(scale_m, cache_b_qs[2]);
|
| 197 |
+
|
| 198 |
+
sum_d += dotPacked4x8EXT(qs_a.w, cache_b_qs[3]) * (scale & 0xF);
|
| 199 |
+
sum_m += dotPacked4x8EXT(scale_m, cache_b_qs[3]);
|
| 200 |
+
|
| 201 |
+
return FLOAT_TYPE(float(cache_b_ds.x) * (float(dm.x) * float(sum_d) - float(dm.y) * float(sum_m)));
|
| 202 |
+
}
|
| 203 |
+
#endif
|
| 204 |
+
|
| 205 |
+
#if defined(DATA_A_Q3_K)
|
| 206 |
+
// 2-byte loads for Q3_K blocks (110 bytes)
|
| 207 |
+
i32vec4 repack4(uint ib, uint iqs) {
|
| 208 |
+
const uint ib_k = ib / 8;
|
| 209 |
+
const uint iqs_k = (ib % 8) * 8 + iqs;
|
| 210 |
+
|
| 211 |
+
const uint qs_idx = (iqs_k / 32) * 8 + (iqs_k % 8);
|
| 212 |
+
const uint qs_shift = ((iqs_k % 32) / 8) * 2;
|
| 213 |
+
const uint hm_shift = iqs_k / 8;
|
| 214 |
+
|
| 215 |
+
const uvec4 qs = uvec4( uint32_t(data_a_packed16[ib_k].qs[qs_idx * 2 ]) |
|
| 216 |
+
(uint32_t(data_a_packed16[ib_k].qs[qs_idx * 2 + 1]) << 16),
|
| 217 |
+
uint32_t(data_a_packed16[ib_k].qs[qs_idx * 2 + 2]) |
|
| 218 |
+
(uint32_t(data_a_packed16[ib_k].qs[qs_idx * 2 + 3]) << 16),
|
| 219 |
+
uint32_t(data_a_packed16[ib_k].qs[qs_idx * 2 + 4]) |
|
| 220 |
+
(uint32_t(data_a_packed16[ib_k].qs[qs_idx * 2 + 5]) << 16),
|
| 221 |
+
uint32_t(data_a_packed16[ib_k].qs[qs_idx * 2 + 6]) |
|
| 222 |
+
(uint32_t(data_a_packed16[ib_k].qs[qs_idx * 2 + 7]) << 16));
|
| 223 |
+
|
| 224 |
+
const uvec4 hmask = uvec4( uint32_t(data_a_packed16[ib_k].hmask[iqs * 2 ]) |
|
| 225 |
+
(uint32_t(data_a_packed16[ib_k].hmask[iqs * 2 + 1]) << 16),
|
| 226 |
+
uint32_t(data_a_packed16[ib_k].hmask[iqs * 2 + 2]) |
|
| 227 |
+
(uint32_t(data_a_packed16[ib_k].hmask[iqs * 2 + 3]) << 16),
|
| 228 |
+
uint32_t(data_a_packed16[ib_k].hmask[iqs * 2 + 4]) |
|
| 229 |
+
(uint32_t(data_a_packed16[ib_k].hmask[iqs * 2 + 5]) << 16),
|
| 230 |
+
uint32_t(data_a_packed16[ib_k].hmask[iqs * 2 + 6]) |
|
| 231 |
+
(uint32_t(data_a_packed16[ib_k].hmask[iqs * 2 + 7]) << 16));
|
| 232 |
+
|
| 233 |
+
// bitwise OR to add 4 if hmask is set, subtract later
|
| 234 |
+
const uint vals0 = (( qs.x >> qs_shift) & 0x03030303) |
|
| 235 |
+
(((hmask.x >> hm_shift) & 0x01010101) << 2);
|
| 236 |
+
const uint vals1 = (( qs.y >> qs_shift) & 0x03030303) |
|
| 237 |
+
(((hmask.y >> hm_shift) & 0x01010101) << 2);
|
| 238 |
+
const uint vals2 = (( qs.z >> qs_shift) & 0x03030303) |
|
| 239 |
+
(((hmask.z >> hm_shift) & 0x01010101) << 2);
|
| 240 |
+
const uint vals3 = (( qs.w >> qs_shift) & 0x03030303) |
|
| 241 |
+
(((hmask.w >> hm_shift) & 0x01010101) << 2);
|
| 242 |
+
|
| 243 |
+
// Subtract 4 by twiddling bits rather than using re-packing as mesa
|
| 244 |
+
// compiles repacking poorly.
|
| 245 |
+
return i32vec4(int32_t(((vals0 ^ 0x80808080) - 0x04040404) ^ 0x80808080),
|
| 246 |
+
int32_t(((vals1 ^ 0x80808080) - 0x04040404) ^ 0x80808080),
|
| 247 |
+
int32_t(((vals2 ^ 0x80808080) - 0x04040404) ^ 0x80808080),
|
| 248 |
+
int32_t(((vals3 ^ 0x80808080) - 0x04040404) ^ 0x80808080));
|
| 249 |
+
}
|
| 250 |
+
|
| 251 |
+
float get_d_scale(uint ib, uint iqs) {
|
| 252 |
+
const uint ib_k = ib / 8;
|
| 253 |
+
const uint iqs_k = (ib % 8) * 8 + iqs;
|
| 254 |
+
const uint is = iqs_k / 4;
|
| 255 |
+
|
| 256 |
+
const int8_t scale = int8_t(((data_a[ib_k].scales[is % 8 ] >> (4 * (is / 8))) & 0x0F0F) |
|
| 257 |
+
(((data_a[ib_k].scales[8 + (is % 4)] >> (2 * (is / 4))) & 0x0303) << 4));
|
| 258 |
+
return float(data_a[ib_k].d) * float(scale - 32);
|
| 259 |
+
}
|
| 260 |
+
|
| 261 |
+
FLOAT_TYPE mmvq_dot_product(const uint ib_a, const uint iqs) {
|
| 262 |
+
int32_t q_sum = 0;
|
| 263 |
+
|
| 264 |
+
const i32vec4 qs_a = repack4(ib_a, iqs * 4);
|
| 265 |
+
const float d_scale = get_d_scale(ib_a, iqs * 4);
|
| 266 |
+
|
| 267 |
+
q_sum += dotPacked4x8EXT(qs_a.x, cache_b_qs[0]);
|
| 268 |
+
q_sum += dotPacked4x8EXT(qs_a.y, cache_b_qs[1]);
|
| 269 |
+
q_sum += dotPacked4x8EXT(qs_a.z, cache_b_qs[2]);
|
| 270 |
+
q_sum += dotPacked4x8EXT(qs_a.w, cache_b_qs[3]);
|
| 271 |
+
|
| 272 |
+
return FLOAT_TYPE(float(cache_b_ds.x) * d_scale * float(q_sum));
|
| 273 |
+
}
|
| 274 |
+
#endif
|
| 275 |
+
|
| 276 |
+
#if defined(DATA_A_Q4_K) || defined(DATA_A_Q5_K)
|
| 277 |
+
// 4-byte loads for Q4_K blocks (144 bytes) and Q5_K blocks (176 bytes)
|
| 278 |
+
i32vec4 repack4(uint ib, uint iqs) {
|
| 279 |
+
const uint ib_k = ib / 8;
|
| 280 |
+
const uint iqs_k = (ib % 8) * 8 + iqs;
|
| 281 |
+
|
| 282 |
+
const uint qs_idx = (iqs_k / 16) * 8 + (iqs_k % 8);
|
| 283 |
+
const uint qs_shift = ((iqs_k % 16) / 8) * 4;
|
| 284 |
+
|
| 285 |
+
#if defined(DATA_A_Q4_K)
|
| 286 |
+
const uint32_t vals0 = (data_a_packed32[ib_k].qs[qs_idx ] >> qs_shift) & 0x0F0F0F0F;
|
| 287 |
+
const uint32_t vals1 = (data_a_packed32[ib_k].qs[qs_idx + 1] >> qs_shift) & 0x0F0F0F0F;
|
| 288 |
+
const uint32_t vals2 = (data_a_packed32[ib_k].qs[qs_idx + 2] >> qs_shift) & 0x0F0F0F0F;
|
| 289 |
+
const uint32_t vals3 = (data_a_packed32[ib_k].qs[qs_idx + 3] >> qs_shift) & 0x0F0F0F0F;
|
| 290 |
+
|
| 291 |
+
return i32vec4(vals0, vals1, vals2, vals3);
|
| 292 |
+
#else // defined(DATA_A_Q5_K)
|
| 293 |
+
const uint qh_idx = iqs;
|
| 294 |
+
const uint qh_shift = iqs_k / 8;
|
| 295 |
+
|
| 296 |
+
return i32vec4(((data_a_packed32[ib_k].qs[qs_idx ] >> qs_shift) & 0x0F0F0F0F) |
|
| 297 |
+
(((data_a_packed32[ib_k].qh[qh_idx ] >> qh_shift) & 0x01010101) << 4),
|
| 298 |
+
((data_a_packed32[ib_k].qs[qs_idx + 1] >> qs_shift) & 0x0F0F0F0F) |
|
| 299 |
+
(((data_a_packed32[ib_k].qh[qh_idx + 1] >> qh_shift) & 0x01010101) << 4),
|
| 300 |
+
((data_a_packed32[ib_k].qs[qs_idx + 2] >> qs_shift) & 0x0F0F0F0F) |
|
| 301 |
+
(((data_a_packed32[ib_k].qh[qh_idx + 2] >> qh_shift) & 0x01010101) << 4),
|
| 302 |
+
((data_a_packed32[ib_k].qs[qs_idx + 3] >> qs_shift) & 0x0F0F0F0F) |
|
| 303 |
+
(((data_a_packed32[ib_k].qh[qh_idx + 3] >> qh_shift) & 0x01010101) << 4));
|
| 304 |
+
#endif
|
| 305 |
+
}
|
| 306 |
+
|
| 307 |
+
vec2 get_dm_scale(uint ib, uint iqs) {
|
| 308 |
+
const uint ib_k = ib / 8;
|
| 309 |
+
const uint iqs_k = (ib % 8) * 8 + iqs;
|
| 310 |
+
const uint is = iqs_k / 8;
|
| 311 |
+
|
| 312 |
+
const uvec3 scales = uvec3(data_a_packed32[ib_k].scales[0],
|
| 313 |
+
data_a_packed32[ib_k].scales[1],
|
| 314 |
+
data_a_packed32[ib_k].scales[2]);
|
| 315 |
+
const uint scalesoffs = (is & 3) * 8;
|
| 316 |
+
|
| 317 |
+
const uint scidx0 = (is < 4) ? 0 : 2;
|
| 318 |
+
const uint scidxshift0 = scalesoffs;
|
| 319 |
+
const uint scidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2;
|
| 320 |
+
const uint mbidx0 = (is < 4) ? 1 : 2;
|
| 321 |
+
const uint mbidxshift0 = (is < 4) ? scalesoffs : scalesoffs + 4;
|
| 322 |
+
const uint mbidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2;
|
| 323 |
+
|
| 324 |
+
const uint8_t sc = uint8_t(((scales[scidx0] >> scidxshift0) & 0xF) | ((scales[0] >> scidxshift1) & 0x30));
|
| 325 |
+
const uint8_t mbyte = uint8_t(((scales[mbidx0] >> mbidxshift0) & 0xF) | ((scales[1] >> mbidxshift1) & 0x30));
|
| 326 |
+
u8vec2 scale_dm = u8vec2(sc, mbyte);
|
| 327 |
+
|
| 328 |
+
return FLOAT_TYPEV2(data_a_packed32[ib_k].dm) * FLOAT_TYPEV2(scale_dm);
|
| 329 |
+
}
|
| 330 |
+
|
| 331 |
+
FLOAT_TYPE mmvq_dot_product(const uint ib_a, const uint iqs) {
|
| 332 |
+
int32_t q_sum = 0;
|
| 333 |
+
|
| 334 |
+
const i32vec4 qs_a = repack4(ib_a, iqs * 4);
|
| 335 |
+
const vec2 dm_scale = get_dm_scale(ib_a, iqs * 4);
|
| 336 |
+
|
| 337 |
+
q_sum += dotPacked4x8EXT(qs_a.x, cache_b_qs[0]);
|
| 338 |
+
q_sum += dotPacked4x8EXT(qs_a.y, cache_b_qs[1]);
|
| 339 |
+
q_sum += dotPacked4x8EXT(qs_a.z, cache_b_qs[2]);
|
| 340 |
+
q_sum += dotPacked4x8EXT(qs_a.w, cache_b_qs[3]);
|
| 341 |
+
|
| 342 |
+
return FLOAT_TYPE(float(cache_b_ds.x) * float(dm_scale.x) * float(q_sum) - float(dm_scale.y) * float(cache_b_ds.y / 2));
|
| 343 |
+
}
|
| 344 |
+
#endif
|
| 345 |
+
|
| 346 |
+
#if defined(DATA_A_Q6_K)
|
| 347 |
+
// 2-byte loads for Q6_K blocks (210 bytes)
|
| 348 |
+
i32vec4 repack4(uint ib, uint iqs) {
|
| 349 |
+
const uint ib_k = ib / 8;
|
| 350 |
+
const uint iqs_k = (ib % 8) * 8 + iqs;
|
| 351 |
+
|
| 352 |
+
const uint ql_idx = (iqs_k / 32) * 16 + iqs_k % 16;
|
| 353 |
+
const uint ql_shift = ((iqs_k % 32) / 16) * 4;
|
| 354 |
+
|
| 355 |
+
const uint qh_idx = (iqs_k / 32) * 8 + iqs;
|
| 356 |
+
const uint qh_shift = ((iqs_k % 32) / 8) * 2;
|
| 357 |
+
|
| 358 |
+
const uvec4 ql = uvec4( uint32_t(data_a_packed16[ib_k].ql[ql_idx * 2 ]) |
|
| 359 |
+
(uint32_t(data_a_packed16[ib_k].ql[ql_idx * 2 + 1]) << 16),
|
| 360 |
+
uint32_t(data_a_packed16[ib_k].ql[ql_idx * 2 + 2]) |
|
| 361 |
+
(uint32_t(data_a_packed16[ib_k].ql[ql_idx * 2 + 3]) << 16),
|
| 362 |
+
uint32_t(data_a_packed16[ib_k].ql[ql_idx * 2 + 4]) |
|
| 363 |
+
(uint32_t(data_a_packed16[ib_k].ql[ql_idx * 2 + 5]) << 16),
|
| 364 |
+
uint32_t(data_a_packed16[ib_k].ql[ql_idx * 2 + 6]) |
|
| 365 |
+
(uint32_t(data_a_packed16[ib_k].ql[ql_idx * 2 + 7]) << 16));
|
| 366 |
+
|
| 367 |
+
const uvec4 qh = uvec4( uint32_t(data_a_packed16[ib_k].qh[qh_idx * 2 ]) |
|
| 368 |
+
(uint32_t(data_a_packed16[ib_k].qh[qh_idx * 2 + 1]) << 16),
|
| 369 |
+
uint32_t(data_a_packed16[ib_k].qh[qh_idx * 2 + 2]) |
|
| 370 |
+
(uint32_t(data_a_packed16[ib_k].qh[qh_idx * 2 + 3]) << 16),
|
| 371 |
+
uint32_t(data_a_packed16[ib_k].qh[qh_idx * 2 + 4]) |
|
| 372 |
+
(uint32_t(data_a_packed16[ib_k].qh[qh_idx * 2 + 5]) << 16),
|
| 373 |
+
uint32_t(data_a_packed16[ib_k].qh[qh_idx * 2 + 6]) |
|
| 374 |
+
(uint32_t(data_a_packed16[ib_k].qh[qh_idx * 2 + 7]) << 16));
|
| 375 |
+
|
| 376 |
+
const uint vals0 = (( ql.x >> ql_shift) & 0x0F0F0F0F) |
|
| 377 |
+
(((qh.x >> qh_shift) & 0x03030303) << 4);
|
| 378 |
+
const uint vals1 = (( ql.y >> ql_shift) & 0x0F0F0F0F) |
|
| 379 |
+
(((qh.y >> qh_shift) & 0x03030303) << 4);
|
| 380 |
+
const uint vals2 = (( ql.z >> ql_shift) & 0x0F0F0F0F) |
|
| 381 |
+
(((qh.z >> qh_shift) & 0x03030303) << 4);
|
| 382 |
+
const uint vals3 = (( ql.w >> ql_shift) & 0x0F0F0F0F) |
|
| 383 |
+
(((qh.w >> qh_shift) & 0x03030303) << 4);
|
| 384 |
+
|
| 385 |
+
// Subtract 32 by twiddling bits rather than using re-packing as mesa
|
| 386 |
+
// compiles repacking poorly.
|
| 387 |
+
return i32vec4(int32_t(((vals0 ^ 0x80808080) - 0x20202020) ^ 0x80808080),
|
| 388 |
+
int32_t(((vals1 ^ 0x80808080) - 0x20202020) ^ 0x80808080),
|
| 389 |
+
int32_t(((vals2 ^ 0x80808080) - 0x20202020) ^ 0x80808080),
|
| 390 |
+
int32_t(((vals3 ^ 0x80808080) - 0x20202020) ^ 0x80808080));
|
| 391 |
+
}
|
| 392 |
+
|
| 393 |
+
float get_d_scale(uint ib, uint iqs) {
|
| 394 |
+
const uint ib_k = ib / 8;
|
| 395 |
+
const uint iqs_k = (ib % 8) * 8 + iqs;
|
| 396 |
+
return float(data_a[ib_k].d) * float(data_a[ib_k].scales[iqs_k / 4]);
|
| 397 |
+
}
|
| 398 |
+
|
| 399 |
+
FLOAT_TYPE mmvq_dot_product(const uint ib_a, const uint iqs) {
|
| 400 |
+
int32_t q_sum = 0;
|
| 401 |
+
|
| 402 |
+
const i32vec4 qs_a = repack4(ib_a, iqs * 4);
|
| 403 |
+
const float d_scale = get_d_scale(ib_a, iqs * 4);
|
| 404 |
+
|
| 405 |
+
q_sum += dotPacked4x8EXT(qs_a.x, cache_b_qs[0]);
|
| 406 |
+
q_sum += dotPacked4x8EXT(qs_a.y, cache_b_qs[1]);
|
| 407 |
+
q_sum += dotPacked4x8EXT(qs_a.z, cache_b_qs[2]);
|
| 408 |
+
q_sum += dotPacked4x8EXT(qs_a.w, cache_b_qs[3]);
|
| 409 |
+
|
| 410 |
+
return FLOAT_TYPE(float(cache_b_ds.x) * float(d_scale) * float(q_sum));
|
| 411 |
+
}
|
| 412 |
+
#endif
|
| 413 |
+
|
| 414 |
+
#if defined(DATA_A_IQ1_S)
|
| 415 |
+
void repack8(uint ib, uint iqs, out i32vec4 out0, out i32vec4 out1) {
|
| 416 |
+
const uint ib32 = iqs / 32;
|
| 417 |
+
|
| 418 |
+
const uint qh = data_a[ib].qh[ib32];
|
| 419 |
+
|
| 420 |
+
const uint qs16_0 = data_a_packed16[ib].qs[(4 * ib32 + 0) / 2];
|
| 421 |
+
const uint qs16_1 = data_a_packed16[ib].qs[(4 * ib32 + 2) / 2];
|
| 422 |
+
|
| 423 |
+
const uint qs0 = qs16_0 & 0xFF;
|
| 424 |
+
const uint qs1 = qs16_0 >> 8;
|
| 425 |
+
const uint qs2 = qs16_1 & 0xFF;
|
| 426 |
+
const uint qs3 = qs16_1 >> 8;
|
| 427 |
+
|
| 428 |
+
const uint hi0 = bitfieldExtract(qh, 3 * int(0), 3);
|
| 429 |
+
const uint hi1 = bitfieldExtract(qh, 3 * int(1), 3);
|
| 430 |
+
const uint hi2 = bitfieldExtract(qh, 3 * int(2), 3);
|
| 431 |
+
const uint hi3 = bitfieldExtract(qh, 3 * int(3), 3);
|
| 432 |
+
|
| 433 |
+
const int32_t grid0 = int32_t(iq1s_grid_gpu[qs0 | (hi0 << 8)]);
|
| 434 |
+
const int32_t grid1 = int32_t(iq1s_grid_gpu[qs1 | (hi1 << 8)]);
|
| 435 |
+
const int32_t grid2 = int32_t(iq1s_grid_gpu[qs2 | (hi2 << 8)]);
|
| 436 |
+
const int32_t grid3 = int32_t(iq1s_grid_gpu[qs3 | (hi3 << 8)]);
|
| 437 |
+
|
| 438 |
+
out0 = i32vec4((grid0 >> 0) & 0x0F0F0F0F,
|
| 439 |
+
(grid0 >> 4) & 0x0F0F0F0F,
|
| 440 |
+
(grid1 >> 0) & 0x0F0F0F0F,
|
| 441 |
+
(grid1 >> 4) & 0x0F0F0F0F);
|
| 442 |
+
out1 = i32vec4((grid2 >> 0) & 0x0F0F0F0F,
|
| 443 |
+
(grid2 >> 4) & 0x0F0F0F0F,
|
| 444 |
+
(grid3 >> 0) & 0x0F0F0F0F,
|
| 445 |
+
(grid3 >> 4) & 0x0F0F0F0F);
|
| 446 |
+
}
|
| 447 |
+
|
| 448 |
+
vec2 get_dm(uint ib, uint iqs) {
|
| 449 |
+
const uint ib32 = iqs / 32;
|
| 450 |
+
|
| 451 |
+
const uint qh = data_a[ib].qh[ib32];
|
| 452 |
+
const float delta = ((qh & 0x8000) != 0) ? -IQ1S_DELTA : IQ1S_DELTA;
|
| 453 |
+
|
| 454 |
+
const float d = float(data_a[ib].d);
|
| 455 |
+
const float dl = d * float(2 * bitfieldExtract(qh, 12, 3) + 1);
|
| 456 |
+
|
| 457 |
+
// the -1 cancels out the bias in iq1s_grid_gpu
|
| 458 |
+
return FLOAT_TYPEV2(dl, dl * (delta - 1));
|
| 459 |
+
}
|
| 460 |
+
|
| 461 |
+
FLOAT_TYPE mmvq_dot_product(const uint ib_a, const uint iqs) {
|
| 462 |
+
int32_t q_sum = 0;
|
| 463 |
+
|
| 464 |
+
const uint ib_k = ib_a / 8;
|
| 465 |
+
const uint iqs_k = (ib_a % 8) * 32 + iqs * 32;
|
| 466 |
+
|
| 467 |
+
i32vec4 qs_a0;
|
| 468 |
+
i32vec4 qs_a1;
|
| 469 |
+
repack8(ib_k, iqs_k, qs_a0, qs_a1);
|
| 470 |
+
|
| 471 |
+
const vec2 dm = get_dm(ib_k, iqs_k);
|
| 472 |
+
|
| 473 |
+
q_sum += dotPacked4x8EXT(qs_a0.x, cache_b_qs[0]);
|
| 474 |
+
q_sum += dotPacked4x8EXT(qs_a0.y, cache_b_qs[1]);
|
| 475 |
+
q_sum += dotPacked4x8EXT(qs_a0.z, cache_b_qs[2]);
|
| 476 |
+
q_sum += dotPacked4x8EXT(qs_a0.w, cache_b_qs[3]);
|
| 477 |
+
q_sum += dotPacked4x8EXT(qs_a1.x, cache_b_qs[4]);
|
| 478 |
+
q_sum += dotPacked4x8EXT(qs_a1.y, cache_b_qs[5]);
|
| 479 |
+
q_sum += dotPacked4x8EXT(qs_a1.z, cache_b_qs[6]);
|
| 480 |
+
q_sum += dotPacked4x8EXT(qs_a1.w, cache_b_qs[7]);
|
| 481 |
+
|
| 482 |
+
return FLOAT_TYPE(float(cache_b_ds.x) * float(dm.x) * float(q_sum) + float(dm.y) * float(cache_b_ds.y));
|
| 483 |
+
}
|
| 484 |
+
#endif
|
| 485 |
+
|
| 486 |
+
#if defined(DATA_A_IQ1_M)
|
| 487 |
+
FLOAT_TYPE mmvq_dot_product(const uint ib_a, const uint iqs) {
|
| 488 |
+
const uint ib_k = ib_a / 8;
|
| 489 |
+
const uint iqs_k = (ib_a % 8) * 32 + iqs * 32;
|
| 490 |
+
|
| 491 |
+
const uint ib32 = iqs_k / 32;
|
| 492 |
+
const uint ib64 = ib32 / 2;
|
| 493 |
+
|
| 494 |
+
const uint16_t[4] scales = data_a[ib_k].scales;
|
| 495 |
+
const u16vec4 s = u16vec4(scales[0], scales[1], scales[2], scales[3]) >> 12;
|
| 496 |
+
const float d = float(unpackHalf2x16(s.x | (s.y << 4) | (s.z << 8) | (s.w << 12)).x);
|
| 497 |
+
|
| 498 |
+
const uint qs32 = data_a_packed32[ib_k].qs[ib32];
|
| 499 |
+
const uint qh16 = data_a_packed16[ib_k].qh[ib32];
|
| 500 |
+
|
| 501 |
+
float sum = 0;
|
| 502 |
+
const uint sc = data_a[ib_k].scales[ib64];
|
| 503 |
+
[[unroll]] for (int l = 0; l < 4; ++l) {
|
| 504 |
+
const uint ib16 = 2 * ib32 + l / 2;
|
| 505 |
+
const float dl = d * (2 * bitfieldExtract(sc, 3 * int(ib16 & 3), 3) + 1);
|
| 506 |
+
const uint qh = qh16 >> (4 * l);
|
| 507 |
+
const uint qs = (qs32 >> (8 * l)) & 0xFF;
|
| 508 |
+
const float delta = ((qh & 8) != 0) ? -IQ1M_DELTA : IQ1M_DELTA;
|
| 509 |
+
|
| 510 |
+
const int32_t grid = int32_t(iq1s_grid_gpu[qs | ((qh & 7) << 8)]);
|
| 511 |
+
|
| 512 |
+
int32_t q_sum = 0;
|
| 513 |
+
q_sum += dotPacked4x8EXT((grid >> 0) & 0x0F0F0F0F, cache_b_qs[2 * l + 0]);
|
| 514 |
+
q_sum += dotPacked4x8EXT((grid >> 4) & 0x0F0F0F0F, cache_b_qs[2 * l + 1]);
|
| 515 |
+
|
| 516 |
+
int32_t y_sum = 0;
|
| 517 |
+
y_sum += dotPacked4x8EXT(int(0x01010101), cache_b_qs[2 * l + 0]);
|
| 518 |
+
y_sum += dotPacked4x8EXT(int(0x01010101), cache_b_qs[2 * l + 1]);
|
| 519 |
+
|
| 520 |
+
// the -1 cancels out the bias in iq1s_grid_gpu
|
| 521 |
+
sum += dl * (q_sum + y_sum * (delta - 1));
|
| 522 |
+
}
|
| 523 |
+
sum *= float(cache_b_ds.x);
|
| 524 |
+
|
| 525 |
+
return sum;
|
| 526 |
+
}
|
| 527 |
+
#endif
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm.comp
ADDED
|
@@ -0,0 +1,466 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
|
| 3 |
+
#extension GL_EXT_control_flow_attributes : enable
|
| 4 |
+
#extension GL_EXT_shader_16bit_storage : require
|
| 5 |
+
|
| 6 |
+
#ifdef FLOAT16
|
| 7 |
+
#extension GL_EXT_shader_explicit_arithmetic_types_float16 : require
|
| 8 |
+
#endif
|
| 9 |
+
#if defined(DATA_A_IQ1_M)
|
| 10 |
+
#extension GL_EXT_shader_explicit_arithmetic_types_int16 : require
|
| 11 |
+
#endif
|
| 12 |
+
|
| 13 |
+
#if defined(DATA_A_BF16) && defined(COOPMAT)
|
| 14 |
+
#extension GL_EXT_bfloat16 : enable
|
| 15 |
+
#endif
|
| 16 |
+
|
| 17 |
+
#ifdef COOPMAT
|
| 18 |
+
#extension GL_KHR_cooperative_matrix : enable
|
| 19 |
+
#extension GL_KHR_memory_scope_semantics : enable
|
| 20 |
+
#endif
|
| 21 |
+
|
| 22 |
+
#if defined(COOPMAT) || defined(MUL_MAT_ID_USE_SUBGROUPS)
|
| 23 |
+
#extension GL_KHR_shader_subgroup_basic : enable
|
| 24 |
+
#extension GL_KHR_shader_subgroup_ballot : enable
|
| 25 |
+
#endif
|
| 26 |
+
|
| 27 |
+
#ifdef MUL_MAT_ID
|
| 28 |
+
#extension GL_EXT_shader_explicit_arithmetic_types_int16 : require
|
| 29 |
+
#endif
|
| 30 |
+
|
| 31 |
+
#include "types.glsl"
|
| 32 |
+
#include "dot_product_funcs.glsl"
|
| 33 |
+
|
| 34 |
+
#ifndef LOAD_VEC_A
|
| 35 |
+
#define LOAD_VEC_A 1
|
| 36 |
+
#endif
|
| 37 |
+
#ifndef LOAD_VEC_B
|
| 38 |
+
#define LOAD_VEC_B 1
|
| 39 |
+
#endif
|
| 40 |
+
|
| 41 |
+
layout (constant_id = 11) const uint ALIGNED = 0;
|
| 42 |
+
|
| 43 |
+
#if !defined(TO_FLOAT_TYPE)
|
| 44 |
+
#define TO_FLOAT_TYPE FLOAT_TYPE
|
| 45 |
+
#endif
|
| 46 |
+
|
| 47 |
+
layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
|
| 48 |
+
|
| 49 |
+
layout (binding = 0) readonly buffer A {A_TYPE data_a[];};
|
| 50 |
+
#if defined(DATA_A_F32)
|
| 51 |
+
layout (binding = 0) readonly buffer A_SCALAR {float data_a_scalar[];};
|
| 52 |
+
#elif defined(DATA_A_F16)
|
| 53 |
+
layout (binding = 0) readonly buffer A_SCALAR {float16_t data_a_scalar[];};
|
| 54 |
+
#elif defined(DATA_A_BF16)
|
| 55 |
+
layout (binding = 0) readonly buffer A_SCALAR {uint16_t data_a_scalar[];};
|
| 56 |
+
#endif
|
| 57 |
+
#if defined(A_TYPE_PACKED16)
|
| 58 |
+
layout (binding = 0) readonly buffer A_PACKED16 {A_TYPE_PACKED16 data_a_packed16[];};
|
| 59 |
+
#endif
|
| 60 |
+
#if defined(A_TYPE_PACKED32)
|
| 61 |
+
layout (binding = 0) readonly buffer A_PACKED32 {A_TYPE_PACKED32 data_a_packed32[];};
|
| 62 |
+
#endif
|
| 63 |
+
|
| 64 |
+
layout (binding = 1) readonly buffer B {B_TYPE data_b[];};
|
| 65 |
+
layout (binding = 1) readonly buffer B_SCALAR {B_TYPE_SCALAR data_b_scalar[];};
|
| 66 |
+
layout (binding = 2) writeonly buffer D {D_TYPE data_d[];};
|
| 67 |
+
|
| 68 |
+
#ifdef MUL_MAT_ID
|
| 69 |
+
layout (binding = 3) readonly buffer IDS {int data_ids[];};
|
| 70 |
+
layout (binding = 4) readonly buffer Counts {int data_expert_count[];};
|
| 71 |
+
#endif
|
| 72 |
+
|
| 73 |
+
layout (push_constant) uniform parameter
|
| 74 |
+
{
|
| 75 |
+
uint M;
|
| 76 |
+
uint N;
|
| 77 |
+
uint K;
|
| 78 |
+
uint stride_a;
|
| 79 |
+
uint stride_b;
|
| 80 |
+
uint stride_d;
|
| 81 |
+
|
| 82 |
+
uint batch_stride_a;
|
| 83 |
+
uint batch_stride_b;
|
| 84 |
+
uint batch_stride_d;
|
| 85 |
+
|
| 86 |
+
#ifdef MUL_MAT_ID
|
| 87 |
+
uint nei0;
|
| 88 |
+
uint nei1;
|
| 89 |
+
uint nbi1;
|
| 90 |
+
uint ne11;
|
| 91 |
+
#else
|
| 92 |
+
uint base_work_group_z;
|
| 93 |
+
uint num_batches;
|
| 94 |
+
uint k_split;
|
| 95 |
+
uint ne02;
|
| 96 |
+
uint ne12;
|
| 97 |
+
uint broadcast2;
|
| 98 |
+
uint broadcast3;
|
| 99 |
+
#endif
|
| 100 |
+
} p;
|
| 101 |
+
|
| 102 |
+
layout (constant_id = 0) const uint BLOCK_SIZE = 64;
|
| 103 |
+
layout (constant_id = 1) const uint BM = 64;
|
| 104 |
+
layout (constant_id = 2) const uint BN = 64;
|
| 105 |
+
layout (constant_id = 4) const uint WM = 32;
|
| 106 |
+
layout (constant_id = 5) const uint WN = 32;
|
| 107 |
+
layout (constant_id = 6) const uint WMITER = 2;
|
| 108 |
+
layout (constant_id = 7) const uint TM = 4;
|
| 109 |
+
layout (constant_id = 8) const uint TN = 2;
|
| 110 |
+
layout (constant_id = 9) const uint TK = 1; // Only needed for coopmat
|
| 111 |
+
layout (constant_id = 10) const uint WARP = 32;
|
| 112 |
+
|
| 113 |
+
#if defined(DATA_A_F32) || defined(DATA_A_F16)
|
| 114 |
+
#define BK 32
|
| 115 |
+
#define BK_STEP 4
|
| 116 |
+
#else
|
| 117 |
+
layout (constant_id = 3) const uint BK = 16; // Assumed to be 32 if working with a quant
|
| 118 |
+
#define BK_STEP 2
|
| 119 |
+
#endif
|
| 120 |
+
|
| 121 |
+
#ifdef COOPMAT
|
| 122 |
+
#define SHMEM_STRIDE (BK / 2 + 4)
|
| 123 |
+
#else
|
| 124 |
+
#define SHMEM_STRIDE (BK / 2 + 1)
|
| 125 |
+
#endif
|
| 126 |
+
|
| 127 |
+
shared FLOAT_TYPEV2 buf_a[BM * SHMEM_STRIDE];
|
| 128 |
+
shared FLOAT_TYPEV2 buf_b[BN * SHMEM_STRIDE];
|
| 129 |
+
|
| 130 |
+
#define NUM_WARPS (BLOCK_SIZE / WARP)
|
| 131 |
+
|
| 132 |
+
#ifdef COOPMAT
|
| 133 |
+
shared ACC_TYPE coopmat_stage[TM * TN * NUM_WARPS];
|
| 134 |
+
#endif
|
| 135 |
+
|
| 136 |
+
#include "mul_mm_id_funcs.glsl"
|
| 137 |
+
#include "mul_mm_funcs.glsl"
|
| 138 |
+
|
| 139 |
+
void main() {
|
| 140 |
+
const uint ic = gl_WorkGroupID.y;
|
| 141 |
+
|
| 142 |
+
#ifdef MUL_MAT_ID
|
| 143 |
+
const uint expert_idx = gl_WorkGroupID.z;
|
| 144 |
+
if (ic * BN >= data_expert_count[expert_idx]) {
|
| 145 |
+
return;
|
| 146 |
+
}
|
| 147 |
+
#endif
|
| 148 |
+
#ifdef NEEDS_INIT_IQ_SHMEM
|
| 149 |
+
init_iq_shmem(gl_WorkGroupSize);
|
| 150 |
+
#endif
|
| 151 |
+
|
| 152 |
+
#ifndef MUL_MAT_ID
|
| 153 |
+
const uint batch_idx = gl_WorkGroupID.z + p.base_work_group_z;
|
| 154 |
+
|
| 155 |
+
const uint i13 = batch_idx / p.ne12;
|
| 156 |
+
const uint i12 = batch_idx % p.ne12;
|
| 157 |
+
|
| 158 |
+
const uint i03 = i13 / p.broadcast3;
|
| 159 |
+
const uint i02 = i12 / p.broadcast2;
|
| 160 |
+
|
| 161 |
+
const uint batch_idx_a = i03 * p.ne02 + i02;
|
| 162 |
+
#endif
|
| 163 |
+
|
| 164 |
+
const uint blocks_m = (p.M + BM - 1) / BM;
|
| 165 |
+
const uint ir = gl_WorkGroupID.x % blocks_m;
|
| 166 |
+
const uint ik = gl_WorkGroupID.x / blocks_m;
|
| 167 |
+
|
| 168 |
+
const uint WNITER = (WM * WN) / (WARP * TM * TN * WMITER);
|
| 169 |
+
const uint WSUBM = WM / WMITER;
|
| 170 |
+
const uint WSUBN = WN / WNITER;
|
| 171 |
+
|
| 172 |
+
#ifdef COOPMAT
|
| 173 |
+
const uint warp_i = gl_SubgroupID;
|
| 174 |
+
|
| 175 |
+
const uint tiw = gl_SubgroupInvocationID;
|
| 176 |
+
|
| 177 |
+
const uint cms_per_row = WM / TM;
|
| 178 |
+
const uint cms_per_col = WN / TN;
|
| 179 |
+
|
| 180 |
+
const uint storestride = WARP / TM;
|
| 181 |
+
const uint store_r = tiw % TM;
|
| 182 |
+
const uint store_c = tiw / TM;
|
| 183 |
+
#else
|
| 184 |
+
const uint warp_i = gl_LocalInvocationID.x / WARP;
|
| 185 |
+
|
| 186 |
+
const uint tiw = gl_LocalInvocationID.x % WARP;
|
| 187 |
+
|
| 188 |
+
const uint tiwr = tiw % (WSUBM / TM);
|
| 189 |
+
const uint tiwc = tiw / (WSUBM / TM);
|
| 190 |
+
#endif
|
| 191 |
+
|
| 192 |
+
const uint warp_r = warp_i % (BM / WM);
|
| 193 |
+
const uint warp_c = warp_i / (BM / WM);
|
| 194 |
+
|
| 195 |
+
#if defined(DATA_A_F32) || defined(DATA_A_F16) || defined(DATA_A_BF16)
|
| 196 |
+
const uint LOAD_VEC_A_EFF = (ALIGNED != 0) ? LOAD_VEC_A : 1;
|
| 197 |
+
const uint LOAD_VEC_BATCH_A = (ALIGNED != 0) ? 1 : 2;
|
| 198 |
+
#else
|
| 199 |
+
const uint LOAD_VEC_A_EFF = LOAD_VEC_A;
|
| 200 |
+
const uint LOAD_VEC_BATCH_A = 1;
|
| 201 |
+
#endif
|
| 202 |
+
const uint LOAD_VEC_B_EFF = (ALIGNED != 0) ? LOAD_VEC_B : 1;
|
| 203 |
+
const uint LOAD_VEC_BATCH_B = (ALIGNED != 0) ? 1 : 2;
|
| 204 |
+
|
| 205 |
+
const uint loadr_a = gl_LocalInvocationID.x % (BK / LOAD_VEC_A_EFF / LOAD_VEC_BATCH_A);
|
| 206 |
+
const uint loadc_a = gl_LocalInvocationID.x / (BK / LOAD_VEC_A_EFF / LOAD_VEC_BATCH_A);
|
| 207 |
+
const uint loadr_b = gl_LocalInvocationID.x % (BK / LOAD_VEC_B_EFF / LOAD_VEC_BATCH_B);
|
| 208 |
+
const uint loadc_b = gl_LocalInvocationID.x / (BK / LOAD_VEC_B_EFF / LOAD_VEC_BATCH_B);
|
| 209 |
+
|
| 210 |
+
const uint loadstride_a = gl_WorkGroupSize.x * LOAD_VEC_A_EFF * LOAD_VEC_BATCH_A / BK;
|
| 211 |
+
const uint loadstride_b = gl_WorkGroupSize.x * LOAD_VEC_B_EFF * LOAD_VEC_BATCH_B / BK;
|
| 212 |
+
|
| 213 |
+
#ifdef MUL_MAT_ID
|
| 214 |
+
#ifdef MUL_MAT_ID_USE_SUBGROUPS
|
| 215 |
+
if (bitCount(p.nei0) == 1) {
|
| 216 |
+
load_row_ids(expert_idx, true, ic);
|
| 217 |
+
} else {
|
| 218 |
+
load_row_ids(expert_idx, false, ic);
|
| 219 |
+
}
|
| 220 |
+
#else
|
| 221 |
+
_ne1 = 0;
|
| 222 |
+
for (uint ii1 = 0; ii1 < p.nei1 && _ne1 < (ic + 1) * BN; ii1++) {
|
| 223 |
+
for (uint ii0 = 0; ii0 < p.nei0 && _ne1 < (ic + 1) * BN; ii0++) {
|
| 224 |
+
if (data_ids[ii1*p.nbi1 + ii0] == expert_idx) {
|
| 225 |
+
if (_ne1 >= ic * BN) {
|
| 226 |
+
row_ids[_ne1 - ic * BN] = u16vec2(ii0, ii1);
|
| 227 |
+
}
|
| 228 |
+
_ne1++;
|
| 229 |
+
}
|
| 230 |
+
}
|
| 231 |
+
}
|
| 232 |
+
|
| 233 |
+
barrier();
|
| 234 |
+
#endif
|
| 235 |
+
|
| 236 |
+
// Workgroup has no work
|
| 237 |
+
if (ic * BN >= _ne1) return;
|
| 238 |
+
#endif
|
| 239 |
+
|
| 240 |
+
#ifdef MUL_MAT_ID
|
| 241 |
+
const uint start_k = 0;
|
| 242 |
+
const uint end_k = p.K;
|
| 243 |
+
#else
|
| 244 |
+
const uint start_k = ik * p.k_split;
|
| 245 |
+
const uint end_k = min(p.K, (ik + 1) * p.k_split);
|
| 246 |
+
#endif
|
| 247 |
+
|
| 248 |
+
uint pos_a =
|
| 249 |
+
#ifdef MUL_MAT_ID
|
| 250 |
+
expert_idx * (p.batch_stride_a / LOAD_VEC_A_EFF) +
|
| 251 |
+
#else
|
| 252 |
+
batch_idx_a * (p.batch_stride_a / LOAD_VEC_A_EFF) +
|
| 253 |
+
#endif
|
| 254 |
+
(ir * BM * p.stride_a + start_k) / LOAD_VEC_A_EFF;
|
| 255 |
+
#ifdef MUL_MAT_ID
|
| 256 |
+
uint pos_b = 0;
|
| 257 |
+
#else
|
| 258 |
+
uint pos_b = (batch_idx * p.batch_stride_b + ic * BN * p.stride_b + start_k) / LOAD_VEC_B_EFF;
|
| 259 |
+
#endif
|
| 260 |
+
|
| 261 |
+
#ifdef COOPMAT
|
| 262 |
+
coopmat<FLOAT_TYPE, gl_ScopeSubgroup, TM, TK, gl_MatrixUseA> cache_a;
|
| 263 |
+
coopmat<FLOAT_TYPE, gl_ScopeSubgroup, TK, TN, gl_MatrixUseB> cache_b;
|
| 264 |
+
coopmat<ACC_TYPE, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator> sums[cms_per_row * cms_per_col];
|
| 265 |
+
|
| 266 |
+
[[unroll]] for (uint i = 0; i < cms_per_row * cms_per_col; i++) {
|
| 267 |
+
sums[i] = coopmat<ACC_TYPE, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator>(0.0f);
|
| 268 |
+
}
|
| 269 |
+
#else
|
| 270 |
+
ACC_TYPEV2 sums[WMITER * TM * WNITER * TN/2];
|
| 271 |
+
#if defined(DATA_A_F32) || defined(DATA_A_F16)
|
| 272 |
+
FLOAT_TYPEV4 cache_a[WMITER * TM];
|
| 273 |
+
FLOAT_TYPEV4 cache_b;
|
| 274 |
+
#else
|
| 275 |
+
FLOAT_TYPEV2 cache_a[WMITER * TM];
|
| 276 |
+
FLOAT_TYPEV2 cache_b;
|
| 277 |
+
#endif
|
| 278 |
+
|
| 279 |
+
[[unroll]] for (uint i = 0; i < WMITER*TM*WNITER*TN/2; i++) {
|
| 280 |
+
sums[i] = ACC_TYPEV2(0.0f, 0.0f);
|
| 281 |
+
}
|
| 282 |
+
#endif
|
| 283 |
+
|
| 284 |
+
for (uint block = start_k; block < end_k; block += BK) {
|
| 285 |
+
[[unroll]] for (uint l = 0; l < BM; l += loadstride_a) {
|
| 286 |
+
load_a_to_shmem(pos_a, loadr_a, loadc_a + l, ir * BM + loadc_a + l, block, end_k);
|
| 287 |
+
}
|
| 288 |
+
[[unroll]] for (uint l = 0; l < BN; l += loadstride_b) {
|
| 289 |
+
#if !defined(MUL_MAT_ID)
|
| 290 |
+
load_b_to_shmem(pos_b, loadr_b, loadc_b + l, ic * BN + loadc_b + l, block, end_k);
|
| 291 |
+
#else
|
| 292 |
+
load_b_to_shmem(pos_b, loadr_b, loadc_b + l, ic, _ne1, block, end_k);
|
| 293 |
+
#endif
|
| 294 |
+
}
|
| 295 |
+
|
| 296 |
+
barrier();
|
| 297 |
+
|
| 298 |
+
pos_a += BK / LOAD_VEC_A_EFF;
|
| 299 |
+
pos_b += BK / LOAD_VEC_B_EFF;
|
| 300 |
+
|
| 301 |
+
#ifdef COOPMAT
|
| 302 |
+
[[unroll]] for (uint i = 0; i < BK; i += TK) {
|
| 303 |
+
[[unroll]] for (uint cm_row = 0; cm_row < cms_per_row; cm_row++) {
|
| 304 |
+
// Load from shared into cache
|
| 305 |
+
coopMatLoad(cache_a, buf_a, (warp_r * WM + cm_row * TM) * SHMEM_STRIDE + i / 2, SHMEM_STRIDE, gl_CooperativeMatrixLayoutRowMajor);
|
| 306 |
+
|
| 307 |
+
[[unroll]] for (uint cm_col = 0; cm_col < cms_per_col; cm_col++) {
|
| 308 |
+
coopMatLoad(cache_b, buf_b, (warp_c * WN + cm_col * TN) * SHMEM_STRIDE + i / 2, SHMEM_STRIDE, gl_CooperativeMatrixLayoutColumnMajor);
|
| 309 |
+
|
| 310 |
+
sums[cm_col * cms_per_row + cm_row] = coopMatMulAdd(cache_a, cache_b, sums[cm_col * cms_per_row + cm_row]);
|
| 311 |
+
}
|
| 312 |
+
}
|
| 313 |
+
}
|
| 314 |
+
#else
|
| 315 |
+
[[unroll]] for (uint i = 0; i < BK / BK_STEP; i++) {
|
| 316 |
+
// Load from shared into cache
|
| 317 |
+
[[unroll]] for (uint wsir = 0; wsir < WMITER; wsir++) {
|
| 318 |
+
[[unroll]] for (uint j = 0; j < TM; j++) {
|
| 319 |
+
#if defined(DATA_A_F32) || defined(DATA_A_F16)
|
| 320 |
+
cache_a[wsir * TM + j].xy = buf_a[(warp_r * WM + wsir * WSUBM + tiwr * TM + j) * SHMEM_STRIDE + 2 * i ];
|
| 321 |
+
cache_a[wsir * TM + j].zw = buf_a[(warp_r * WM + wsir * WSUBM + tiwr * TM + j) * SHMEM_STRIDE + 2 * i + 1];
|
| 322 |
+
#else
|
| 323 |
+
cache_a[wsir * TM + j] = buf_a[(warp_r * WM + wsir * WSUBM + tiwr * TM + j) * SHMEM_STRIDE + i];
|
| 324 |
+
#endif
|
| 325 |
+
}
|
| 326 |
+
}
|
| 327 |
+
|
| 328 |
+
[[unroll]] for (uint wsic = 0; wsic < WNITER; wsic++) {
|
| 329 |
+
[[unroll]] for (uint cc = 0; cc < TN; cc++) {
|
| 330 |
+
#if defined(DATA_A_F32) || defined(DATA_A_F16)
|
| 331 |
+
cache_b.xy = buf_b[(warp_c * WN + wsic * WSUBN + tiwc * TN + cc) * SHMEM_STRIDE + 2 * i ];
|
| 332 |
+
cache_b.zw = buf_b[(warp_c * WN + wsic * WSUBN + tiwc * TN + cc) * SHMEM_STRIDE + 2 * i + 1];
|
| 333 |
+
#else
|
| 334 |
+
cache_b = buf_b[(warp_c * WN + wsic * WSUBN + tiwc * TN + cc) * SHMEM_STRIDE + i];
|
| 335 |
+
#endif
|
| 336 |
+
|
| 337 |
+
[[unroll]] for (uint wsir = 0; wsir < WMITER; wsir++) {
|
| 338 |
+
[[unroll]] for (uint cr = 0; cr < TM / 2; cr++) {
|
| 339 |
+
// [WNITER][TN][WMITER][TM / 2] -> [wsic][cc][wsir][cr]
|
| 340 |
+
const uint sums_idx = (wsic * TN + cc) * WMITER * (TM / 2) + wsir * (TM / 2) + cr;
|
| 341 |
+
sums[sums_idx].x = dot_product(cache_a[wsir * TM + 2 * cr ], cache_b, sums[sums_idx].x);
|
| 342 |
+
sums[sums_idx].y = dot_product(cache_a[wsir * TM + 2 * cr + 1], cache_b, sums[sums_idx].y);
|
| 343 |
+
}
|
| 344 |
+
}
|
| 345 |
+
}
|
| 346 |
+
}
|
| 347 |
+
|
| 348 |
+
}
|
| 349 |
+
#endif
|
| 350 |
+
|
| 351 |
+
barrier();
|
| 352 |
+
}
|
| 353 |
+
|
| 354 |
+
#if defined(ACC_TYPE_MAX)
|
| 355 |
+
#ifdef COOPMAT
|
| 356 |
+
[[unroll]] for (uint j = 0; j < cms_per_row * cms_per_col; j++) {
|
| 357 |
+
[[unroll]] for (uint i = 0; i < sums[j].length(); ++i) {
|
| 358 |
+
sums[j][i] = clamp(sums[j][i], -ACC_TYPE_MAX, ACC_TYPE_MAX);
|
| 359 |
+
}
|
| 360 |
+
}
|
| 361 |
+
#else
|
| 362 |
+
[[unroll]] for (uint i = 0; i < WMITER*TM*WNITER*TN/2; i++) {
|
| 363 |
+
sums[i].x = clamp(sums[i].x, -ACC_TYPE_MAX, ACC_TYPE_MAX);
|
| 364 |
+
sums[i].y = clamp(sums[i].y, -ACC_TYPE_MAX, ACC_TYPE_MAX);
|
| 365 |
+
}
|
| 366 |
+
#endif
|
| 367 |
+
#endif
|
| 368 |
+
|
| 369 |
+
const uint dr = ir * BM + warp_r * WM;
|
| 370 |
+
const uint dc = ic * BN + warp_c * WN;
|
| 371 |
+
|
| 372 |
+
#ifndef MUL_MAT_ID
|
| 373 |
+
const uint offsets = batch_idx * p.batch_stride_d + ik * p.batch_stride_d * p.num_batches;
|
| 374 |
+
#endif
|
| 375 |
+
|
| 376 |
+
#ifdef COOPMAT
|
| 377 |
+
#ifdef MUL_MAT_ID
|
| 378 |
+
[[unroll]] for (uint cm_row = 0; cm_row < cms_per_row; cm_row++) {
|
| 379 |
+
[[unroll]] for (uint cm_col = 0; cm_col < cms_per_col; cm_col++) {
|
| 380 |
+
coopMatStore(sums[cm_col * cms_per_row + cm_row], coopmat_stage, warp_i * TM * TN, TM, gl_CooperativeMatrixLayoutColumnMajor);
|
| 381 |
+
|
| 382 |
+
barrier();
|
| 383 |
+
[[unroll]] for (uint col = 0; col < TN; col += storestride) {
|
| 384 |
+
const uint row_i = dc + cm_col * TN + col + store_c;
|
| 385 |
+
if (row_i >= _ne1) break;
|
| 386 |
+
|
| 387 |
+
const u16vec2 row_idx = row_ids[row_i - ic * BN];
|
| 388 |
+
|
| 389 |
+
if (dr + cm_row * TM + store_r < p.M) {
|
| 390 |
+
data_d[row_idx.y * p.batch_stride_d + row_idx.x * p.stride_d + dr + cm_row * TM + store_r] = D_TYPE(coopmat_stage[warp_i * TM * TN + (col + store_c) * TM + store_r]);
|
| 391 |
+
}
|
| 392 |
+
}
|
| 393 |
+
barrier();
|
| 394 |
+
}
|
| 395 |
+
}
|
| 396 |
+
#else
|
| 397 |
+
const bool is_aligned = p.stride_d % 4 == 0; // Assumption: D_TYPE == float
|
| 398 |
+
|
| 399 |
+
[[unroll]] for (uint cm_row = 0; cm_row < cms_per_row; cm_row++) {
|
| 400 |
+
[[unroll]] for (uint cm_col = 0; cm_col < cms_per_col; cm_col++) {
|
| 401 |
+
const bool is_in_bounds = dr + (cm_row + 1) * TM <= p.M && dc + (cm_col + 1) * TN <= p.N;
|
| 402 |
+
|
| 403 |
+
if (is_aligned && is_in_bounds) {
|
| 404 |
+
// Full coopMat is within bounds and stride_d is aligned with 16B
|
| 405 |
+
coopmat<D_TYPE, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator> cm_dtype = coopmat<D_TYPE, gl_ScopeSubgroup, TM, TN, gl_MatrixUseAccumulator>(sums[cm_col * cms_per_row + cm_row]);
|
| 406 |
+
coopMatStore(cm_dtype, data_d, offsets + (dc + cm_col * TN) * p.stride_d + dr + cm_row * TM, p.stride_d, gl_CooperativeMatrixLayoutColumnMajor);
|
| 407 |
+
} else if (is_in_bounds) {
|
| 408 |
+
// Full coopMat is within bounds, but stride_d is not aligned
|
| 409 |
+
coopMatStore(sums[cm_col * cms_per_row + cm_row], coopmat_stage, warp_i * TM * TN, TM, gl_CooperativeMatrixLayoutColumnMajor);
|
| 410 |
+
|
| 411 |
+
controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease);
|
| 412 |
+
[[unroll]] for (uint col = 0; col < TN; col += storestride) {
|
| 413 |
+
data_d[offsets + (dc + cm_col * TN + col + store_c) * p.stride_d + dr + cm_row * TM + store_r] = D_TYPE(coopmat_stage[warp_i * TM * TN + (col + store_c) * TM + store_r]);
|
| 414 |
+
}
|
| 415 |
+
controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease);
|
| 416 |
+
} else if (dr + cm_row * TM < p.M && dc + cm_col * TN < p.N) {
|
| 417 |
+
// Partial coopMat is within bounds
|
| 418 |
+
coopMatStore(sums[cm_col * cms_per_row + cm_row], coopmat_stage, warp_i * TM * TN, TM, gl_CooperativeMatrixLayoutColumnMajor);
|
| 419 |
+
|
| 420 |
+
controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease);
|
| 421 |
+
[[unroll]] for (uint col = 0; col < TN; col += storestride) {
|
| 422 |
+
if (dr + cm_row * TM + store_r < p.M && dc + cm_col * TN + col + store_c < p.N) {
|
| 423 |
+
data_d[offsets + (dc + cm_col * TN + col + store_c) * p.stride_d + dr + cm_row * TM + store_r] = D_TYPE(coopmat_stage[warp_i * TM * TN + (col + store_c) * TM + store_r]);
|
| 424 |
+
}
|
| 425 |
+
}
|
| 426 |
+
controlBarrier(gl_ScopeSubgroup, gl_ScopeSubgroup, gl_StorageSemanticsShared, gl_SemanticsAcquireRelease);
|
| 427 |
+
}
|
| 428 |
+
}
|
| 429 |
+
}
|
| 430 |
+
#endif // MUL_MAT_ID
|
| 431 |
+
#else
|
| 432 |
+
[[unroll]] for (uint wsic = 0; wsic < WNITER; wsic++) {
|
| 433 |
+
[[unroll]] for (uint wsir = 0; wsir < WMITER; wsir++) {
|
| 434 |
+
|
| 435 |
+
const uint dr_warp = dr + wsir * WSUBM + tiwr * TM;
|
| 436 |
+
const uint dc_warp = dc + wsic * WSUBN + tiwc * TN;
|
| 437 |
+
[[unroll]] for (uint cc = 0; cc < TN; cc++) {
|
| 438 |
+
#ifdef MUL_MAT_ID
|
| 439 |
+
const uint row_i = dc_warp + cc;
|
| 440 |
+
if (row_i >= _ne1) break;
|
| 441 |
+
|
| 442 |
+
const u16vec2 row_idx = row_ids[row_i - ic * BN];
|
| 443 |
+
#endif // MUL_MAT_ID
|
| 444 |
+
[[unroll]] for (uint cr = 0; cr < TM / 2; cr++) {
|
| 445 |
+
const uint sums_idx = (wsic * TN + cc) * WMITER * (TM / 2) + wsir * (TM / 2) + cr;
|
| 446 |
+
#ifdef MUL_MAT_ID
|
| 447 |
+
if (dr_warp + 2 * cr < p.M) {
|
| 448 |
+
data_d[row_idx.y * p.batch_stride_d + row_idx.x * p.stride_d + dr_warp + 2 * cr] = D_TYPE(sums[sums_idx].x);
|
| 449 |
+
}
|
| 450 |
+
if (dr_warp + 2 * cr + 1 < p.M) {
|
| 451 |
+
data_d[row_idx.y * p.batch_stride_d + row_idx.x * p.stride_d + dr_warp + 2 * cr + 1] = D_TYPE(sums[sums_idx].y);
|
| 452 |
+
}
|
| 453 |
+
#else
|
| 454 |
+
if (dr_warp + 2 * cr < p.M && dc_warp + cc < p.N) {
|
| 455 |
+
data_d[offsets + (dc_warp + cc) * p.stride_d + dr_warp + 2 * cr] = D_TYPE(sums[sums_idx].x);
|
| 456 |
+
}
|
| 457 |
+
if (dr_warp + 2 * cr + 1 < p.M && dc_warp + cc < p.N) {
|
| 458 |
+
data_d[offsets + (dc_warp + cc) * p.stride_d + dr_warp + 2 * cr + 1] = D_TYPE(sums[sums_idx].y);
|
| 459 |
+
}
|
| 460 |
+
#endif // MUL_MAT_ID
|
| 461 |
+
}
|
| 462 |
+
}
|
| 463 |
+
}
|
| 464 |
+
}
|
| 465 |
+
#endif // COOPMAT
|
| 466 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_cm2.comp
ADDED
|
@@ -0,0 +1,658 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
|
| 3 |
+
#extension GL_EXT_control_flow_attributes : enable
|
| 4 |
+
#extension GL_EXT_shader_16bit_storage : require
|
| 5 |
+
|
| 6 |
+
#extension GL_EXT_shader_explicit_arithmetic_types_float16 : require
|
| 7 |
+
#extension GL_EXT_shader_explicit_arithmetic_types_int8 : require
|
| 8 |
+
#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
|
| 9 |
+
#extension GL_EXT_shader_explicit_arithmetic_types_int16 : require
|
| 10 |
+
|
| 11 |
+
#extension GL_KHR_memory_scope_semantics : enable
|
| 12 |
+
#extension GL_KHR_cooperative_matrix : enable
|
| 13 |
+
#extension GL_NV_cooperative_matrix2 : enable
|
| 14 |
+
#ifdef GGML_VULKAN_COOPMAT2_DECODE_VECTOR
|
| 15 |
+
#extension GL_NV_cooperative_matrix_decode_vector : enable
|
| 16 |
+
#endif
|
| 17 |
+
#extension GL_EXT_buffer_reference : enable
|
| 18 |
+
#extension GL_KHR_shader_subgroup_ballot : enable
|
| 19 |
+
#extension GL_KHR_shader_subgroup_vote : enable
|
| 20 |
+
#ifdef DATA_A_BF16
|
| 21 |
+
#extension GL_EXT_bfloat16 : enable
|
| 22 |
+
#endif
|
| 23 |
+
|
| 24 |
+
#include "types.glsl"
|
| 25 |
+
#include "utils.glsl"
|
| 26 |
+
|
| 27 |
+
layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
|
| 28 |
+
|
| 29 |
+
#define IS_MUL_MM2 1
|
| 30 |
+
|
| 31 |
+
layout (constant_id = 0) const uint BLOCK_SIZE = 256;
|
| 32 |
+
layout (constant_id = 1) const uint BM = 64;
|
| 33 |
+
layout (constant_id = 2) const uint BN = 64;
|
| 34 |
+
layout (constant_id = 3) const uint BK = 16; // Assumed to be 32 if working with a quant
|
| 35 |
+
|
| 36 |
+
layout (constant_id = 4) const bool enable_smaller_matrices = false;
|
| 37 |
+
const uint BNover2 = enable_smaller_matrices ? (BN / 2) : BN;
|
| 38 |
+
const uint BNover4 = enable_smaller_matrices ? (BN / 4) : BN;
|
| 39 |
+
layout (constant_id = 5) const uint ALIGNED = 0;
|
| 40 |
+
|
| 41 |
+
layout (push_constant) uniform parameter
|
| 42 |
+
{
|
| 43 |
+
uint M;
|
| 44 |
+
uint N;
|
| 45 |
+
uint K;
|
| 46 |
+
uint stride_a;
|
| 47 |
+
uint stride_b;
|
| 48 |
+
uint stride_d;
|
| 49 |
+
|
| 50 |
+
uint batch_stride_a;
|
| 51 |
+
uint batch_stride_b;
|
| 52 |
+
uint batch_stride_d;
|
| 53 |
+
|
| 54 |
+
#ifdef MUL_MAT_ID
|
| 55 |
+
uint nei0;
|
| 56 |
+
uint nei1;
|
| 57 |
+
uint nbi1;
|
| 58 |
+
uint ne11;
|
| 59 |
+
#else
|
| 60 |
+
uint base_work_group_z;
|
| 61 |
+
uint num_batches;
|
| 62 |
+
uint k_split;
|
| 63 |
+
uint ne02;
|
| 64 |
+
uint ne12;
|
| 65 |
+
uint broadcast2;
|
| 66 |
+
uint broadcast3;
|
| 67 |
+
#endif
|
| 68 |
+
// N dimension for the B matrix can be >= p.N
|
| 69 |
+
uint padded_N;
|
| 70 |
+
} p;
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
layout (binding = 0) readonly buffer A {A_TYPE data_a[];};
|
| 74 |
+
layout (binding = 1) readonly buffer B {B_TYPE data_b[];};
|
| 75 |
+
layout (binding = 2) writeonly buffer D {D_TYPE data_d[];};
|
| 76 |
+
#if defined(MUL_MAT_ID) && defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR)
|
| 77 |
+
layout (binding = 1) readonly buffer B4 {B_TYPEV4 data_b_v4[];};
|
| 78 |
+
#endif
|
| 79 |
+
|
| 80 |
+
#if QUANT_K > 1
|
| 81 |
+
#include "dequant_funcs_cm2.glsl"
|
| 82 |
+
#if defined(dequantFuncA_v) && defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR)
|
| 83 |
+
#define DECODEFUNCA , dequantFuncA, dequantFuncA_v
|
| 84 |
+
#else
|
| 85 |
+
#define DECODEFUNCA , dequantFuncA
|
| 86 |
+
#endif
|
| 87 |
+
#else
|
| 88 |
+
#define DECODEFUNCA
|
| 89 |
+
#endif
|
| 90 |
+
|
| 91 |
+
#if !defined(fetch_scales)
|
| 92 |
+
#define fetch_scales(a, b, c, d, e, f)
|
| 93 |
+
#endif
|
| 94 |
+
#if !defined(store_scales)
|
| 95 |
+
#define store_scales(a)
|
| 96 |
+
#endif
|
| 97 |
+
|
| 98 |
+
#if defined(DATA_A_BF16)
|
| 99 |
+
#define MAT_TYPE bfloat16_t
|
| 100 |
+
#else
|
| 101 |
+
#define MAT_TYPE FLOAT_TYPE
|
| 102 |
+
#endif
|
| 103 |
+
|
| 104 |
+
#ifdef MUL_MAT_ID
|
| 105 |
+
layout (binding = 3) readonly buffer IDS {int data_ids[];};
|
| 106 |
+
layout (binding = 4) readonly buffer Counts {int data_expert_count[];};
|
| 107 |
+
|
| 108 |
+
shared u16vec4 row_ids[BN];
|
| 109 |
+
|
| 110 |
+
layout(buffer_reference, std430, buffer_reference_align = 2) buffer decodeBufB {
|
| 111 |
+
B_TYPE b[];
|
| 112 |
+
};
|
| 113 |
+
|
| 114 |
+
uint _ne1;
|
| 115 |
+
layout (constant_id = 6) const uint subgroup_size = 32;
|
| 116 |
+
shared uvec4 ballots_sh[BLOCK_SIZE / subgroup_size];
|
| 117 |
+
|
| 118 |
+
B_TYPE decodeFuncB(const in decodeBufB bl, const in uint blockCoords[2], const in uint coordInBlock[2])
|
| 119 |
+
{
|
| 120 |
+
const uint row_i = blockCoords[0];
|
| 121 |
+
|
| 122 |
+
const u16vec4 row_idx = row_ids[row_i];
|
| 123 |
+
#if defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR)
|
| 124 |
+
// The decode-vector path gives B a K-dimension tensor-layout block size of BK.
|
| 125 |
+
const uint k = blockCoords[1] * BK + coordInBlock[1];
|
| 126 |
+
#else
|
| 127 |
+
const uint k = blockCoords[1];
|
| 128 |
+
#endif
|
| 129 |
+
B_TYPE ret = data_b[row_idx.y * p.batch_stride_b + row_idx.x * p.stride_b + k];
|
| 130 |
+
|
| 131 |
+
return ret;
|
| 132 |
+
}
|
| 133 |
+
|
| 134 |
+
#if defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR)
|
| 135 |
+
B_TYPEV4 decodeFuncB_v(const in decodeBufB bl, const in uint blockCoords[2], const in uint coordInBlock[2])
|
| 136 |
+
{
|
| 137 |
+
const uint row_i = blockCoords[0];
|
| 138 |
+
|
| 139 |
+
const u16vec4 row_idx = row_ids[row_i];
|
| 140 |
+
const uint k = blockCoords[1] * BK + coordInBlock[1];
|
| 141 |
+
const uint base = row_idx.y * p.batch_stride_b + row_idx.x * p.stride_b + k;
|
| 142 |
+
|
| 143 |
+
return data_b_v4[base >> 2];
|
| 144 |
+
}
|
| 145 |
+
#define DECODEFUNCB , decodeFuncB, decodeFuncB_v
|
| 146 |
+
#else
|
| 147 |
+
#define DECODEFUNCB , decodeFuncB
|
| 148 |
+
#endif
|
| 149 |
+
|
| 150 |
+
D_TYPE perElemOpD(const in uint32_t r, const in uint32_t c, const in D_TYPE elem, const in uint32_t ir, const in uint32_t ic)
|
| 151 |
+
{
|
| 152 |
+
uint dr = ir * BM + r;
|
| 153 |
+
uint dc = ic * BN + c;
|
| 154 |
+
|
| 155 |
+
if (dr < p.M && dc < _ne1) {
|
| 156 |
+
uint row_i = c;
|
| 157 |
+
const u16vec4 row_idx = row_ids[row_i];
|
| 158 |
+
data_d[row_idx.y * p.batch_stride_d + row_idx.z * p.stride_d + dr] = elem;
|
| 159 |
+
}
|
| 160 |
+
return elem;
|
| 161 |
+
}
|
| 162 |
+
|
| 163 |
+
void load_row_ids(uint expert_idx, bool nei0_is_pow2, uint ic) {
|
| 164 |
+
_ne1 = 0;
|
| 165 |
+
uint num_elements = p.nei1 * p.nei0;
|
| 166 |
+
uint nei0shift = findLSB(p.nei0);
|
| 167 |
+
|
| 168 |
+
uint ids[16];
|
| 169 |
+
uint iter = 0;
|
| 170 |
+
|
| 171 |
+
uint expert_count = data_expert_count[expert_idx];
|
| 172 |
+
|
| 173 |
+
for (uint j = 0; j < num_elements; j += BLOCK_SIZE) {
|
| 174 |
+
// prefetch up to 16 elements
|
| 175 |
+
if (iter == 0) {
|
| 176 |
+
[[unroll]] for (uint k = 0; k < 16; ++k) {
|
| 177 |
+
uint i = j + gl_LocalInvocationIndex + k*BLOCK_SIZE;
|
| 178 |
+
bool in_range = i < num_elements;
|
| 179 |
+
uint ii1;
|
| 180 |
+
if (nei0_is_pow2) {
|
| 181 |
+
ii1 = i >> nei0shift;
|
| 182 |
+
} else {
|
| 183 |
+
ii1 = i / p.nei0;
|
| 184 |
+
}
|
| 185 |
+
uint ii0 = i - ii1 * p.nei0;
|
| 186 |
+
ids[k] = in_range ? data_ids[ii1*p.nbi1 + ii0] : 0;
|
| 187 |
+
}
|
| 188 |
+
}
|
| 189 |
+
uint i = j + gl_LocalInvocationIndex;
|
| 190 |
+
bool in_range = i < num_elements;
|
| 191 |
+
uint ii1;
|
| 192 |
+
if (nei0_is_pow2) {
|
| 193 |
+
ii1 = i >> nei0shift;
|
| 194 |
+
} else {
|
| 195 |
+
ii1 = i / p.nei0;
|
| 196 |
+
}
|
| 197 |
+
uint ii0 = i - ii1 * p.nei0;
|
| 198 |
+
uint id = ids[iter++];
|
| 199 |
+
uvec4 ballot = subgroupBallot(in_range && id == expert_idx);
|
| 200 |
+
|
| 201 |
+
if (gl_SubgroupInvocationID == 0) {
|
| 202 |
+
ballots_sh[gl_SubgroupID] = ballot;
|
| 203 |
+
}
|
| 204 |
+
barrier();
|
| 205 |
+
|
| 206 |
+
uint subgroup_base = 0;
|
| 207 |
+
uint total = 0;
|
| 208 |
+
for (uint k = 0; k < gl_NumSubgroups; ++k) {
|
| 209 |
+
if (k == gl_SubgroupID) {
|
| 210 |
+
subgroup_base = total;
|
| 211 |
+
}
|
| 212 |
+
total += subgroupBallotBitCount(ballots_sh[k]);
|
| 213 |
+
}
|
| 214 |
+
barrier();
|
| 215 |
+
|
| 216 |
+
uint idx = subgroup_base + subgroupBallotExclusiveBitCount(ballot);
|
| 217 |
+
if (in_range && id == expert_idx && _ne1 + idx >= ic * BN && _ne1 + idx < (ic + 1) * BN) {
|
| 218 |
+
row_ids[_ne1 + idx - ic * BN] = u16vec4(fastmod(ii0, p.ne11), ii1, ii0, 0);
|
| 219 |
+
}
|
| 220 |
+
_ne1 += total;
|
| 221 |
+
iter &= 15;
|
| 222 |
+
if (_ne1 >= (ic + 1) * BN || _ne1 == expert_count) {
|
| 223 |
+
break;
|
| 224 |
+
}
|
| 225 |
+
}
|
| 226 |
+
barrier();
|
| 227 |
+
}
|
| 228 |
+
#endif
|
| 229 |
+
|
| 230 |
+
void main() {
|
| 231 |
+
const uint tid = gl_LocalInvocationIndex;
|
| 232 |
+
const uint ic = gl_WorkGroupID.y;
|
| 233 |
+
|
| 234 |
+
#ifdef MUL_MAT_ID
|
| 235 |
+
const uint expert_idx = gl_WorkGroupID.z;
|
| 236 |
+
if (ic * BN >= data_expert_count[expert_idx]) {
|
| 237 |
+
return;
|
| 238 |
+
}
|
| 239 |
+
// initialize to row 0 so we don't need to bounds check
|
| 240 |
+
if (tid < BN) {
|
| 241 |
+
row_ids[tid] = u16vec4(0);
|
| 242 |
+
}
|
| 243 |
+
#if !defined(NEEDS_INIT_IQ_SHMEM)
|
| 244 |
+
barrier();
|
| 245 |
+
#endif
|
| 246 |
+
#endif
|
| 247 |
+
|
| 248 |
+
#ifdef NEEDS_INIT_IQ_SHMEM
|
| 249 |
+
init_iq_shmem(gl_WorkGroupSize);
|
| 250 |
+
#endif
|
| 251 |
+
|
| 252 |
+
#ifndef MUL_MAT_ID
|
| 253 |
+
const uint batch_idx = gl_WorkGroupID.z + p.base_work_group_z;
|
| 254 |
+
|
| 255 |
+
const uint i13 = batch_idx / p.ne12;
|
| 256 |
+
const uint i12 = batch_idx % p.ne12;
|
| 257 |
+
|
| 258 |
+
const uint i03 = i13 / p.broadcast3;
|
| 259 |
+
const uint i02 = i12 / p.broadcast2;
|
| 260 |
+
|
| 261 |
+
const uint batch_idx_a = i03 * p.ne02 + i02;
|
| 262 |
+
#endif
|
| 263 |
+
|
| 264 |
+
const uint blocks_m = (p.M + BM - 1) / BM;
|
| 265 |
+
const uint ir = gl_WorkGroupID.x % blocks_m;
|
| 266 |
+
const uint ik = gl_WorkGroupID.x / blocks_m;
|
| 267 |
+
|
| 268 |
+
#ifdef MUL_MAT_ID
|
| 269 |
+
if (bitCount(p.nei0) == 1) {
|
| 270 |
+
load_row_ids(expert_idx, true, ic);
|
| 271 |
+
} else {
|
| 272 |
+
load_row_ids(expert_idx, false, ic);
|
| 273 |
+
}
|
| 274 |
+
|
| 275 |
+
// Workgroup has no work
|
| 276 |
+
if (ic * BN >= _ne1) return;
|
| 277 |
+
#endif
|
| 278 |
+
|
| 279 |
+
#ifdef MUL_MAT_ID
|
| 280 |
+
uint start_k = 0;
|
| 281 |
+
const uint end_k = p.K;
|
| 282 |
+
#else
|
| 283 |
+
uint start_k = ik * p.k_split;
|
| 284 |
+
const uint end_k = min(p.K, (ik + 1) * p.k_split);
|
| 285 |
+
#endif
|
| 286 |
+
|
| 287 |
+
#ifdef MUL_MAT_ID
|
| 288 |
+
uint pos_a = expert_idx * (p.batch_stride_a / QUANT_K);
|
| 289 |
+
uint pos_b = 0;
|
| 290 |
+
#else
|
| 291 |
+
uint pos_a = batch_idx_a * (p.batch_stride_a / QUANT_K);
|
| 292 |
+
uint pos_b = batch_idx * p.batch_stride_b;
|
| 293 |
+
uint pos_d = batch_idx * p.batch_stride_d + ik * p.batch_stride_d * p.num_batches;
|
| 294 |
+
#endif
|
| 295 |
+
|
| 296 |
+
uint stride_a = p.stride_a / QUANT_K;
|
| 297 |
+
uint stride_b = p.stride_b;
|
| 298 |
+
|
| 299 |
+
// Hint to the compiler that values are aligned (want 16B alignment).
|
| 300 |
+
// Quants are always block-aligned, no alignment needed.
|
| 301 |
+
if (ALIGNED != 0) {
|
| 302 |
+
#if QUANT_K == 1
|
| 303 |
+
stride_a &= ~7;
|
| 304 |
+
#endif
|
| 305 |
+
stride_b &= ~7;
|
| 306 |
+
}
|
| 307 |
+
|
| 308 |
+
// Create layouts for both clamped and unclamped accesses
|
| 309 |
+
tensorLayoutNV<2> tensorLayoutA = createTensorLayoutNV(2);
|
| 310 |
+
tensorLayoutNV<2, gl_CooperativeMatrixClampModeConstantNV> tensorLayoutAClamp = createTensorLayoutNV(2, gl_CooperativeMatrixClampModeConstantNV);
|
| 311 |
+
tensorLayoutNV<2> tensorLayoutB = createTensorLayoutNV(2);
|
| 312 |
+
tensorLayoutNV<2, gl_CooperativeMatrixClampModeConstantNV> tensorLayoutBClamp = createTensorLayoutNV(2, gl_CooperativeMatrixClampModeConstantNV);
|
| 313 |
+
tensorLayoutNV<2, gl_CooperativeMatrixClampModeConstantNV> tensorLayoutD = createTensorLayoutNV(2, gl_CooperativeMatrixClampModeConstantNV);
|
| 314 |
+
|
| 315 |
+
#if QUANT_K > 1
|
| 316 |
+
tensorLayoutA = setTensorLayoutBlockSizeNV(tensorLayoutA, 1, QUANT_K);
|
| 317 |
+
tensorLayoutAClamp = setTensorLayoutBlockSizeNV(tensorLayoutAClamp, 1, QUANT_K);
|
| 318 |
+
#endif
|
| 319 |
+
#if defined(MUL_MAT_ID) && defined(GGML_VULKAN_COOPMAT2_DECODE_VECTOR)
|
| 320 |
+
tensorLayoutB = setTensorLayoutBlockSizeNV(tensorLayoutB, 1, BK);
|
| 321 |
+
#endif
|
| 322 |
+
|
| 323 |
+
// Use end_k rather than p.K as the dimension because that's what
|
| 324 |
+
// we need to bound check against when using split_k.
|
| 325 |
+
// Bounds check B against padded_N, but bounds check D against N.
|
| 326 |
+
tensorLayoutA = setTensorLayoutDimensionNV(tensorLayoutA, p.M, end_k);
|
| 327 |
+
tensorLayoutB = setTensorLayoutDimensionNV(tensorLayoutB, p.padded_N, end_k);
|
| 328 |
+
tensorLayoutD = setTensorLayoutDimensionNV(tensorLayoutD, p.N, p.M);
|
| 329 |
+
tensorLayoutAClamp = setTensorLayoutDimensionNV(tensorLayoutAClamp, p.M, end_k);
|
| 330 |
+
tensorLayoutBClamp = setTensorLayoutDimensionNV(tensorLayoutBClamp, p.padded_N, end_k);
|
| 331 |
+
|
| 332 |
+
tensorLayoutD = setTensorLayoutStrideNV(tensorLayoutD, p.stride_d, 1);
|
| 333 |
+
|
| 334 |
+
tensorViewNV<2, false, 1, 0> tensorViewTranspose = createTensorViewNV(2, false, 1, 0);
|
| 335 |
+
|
| 336 |
+
#if !defined(MUL_MAT_ID)
|
| 337 |
+
|
| 338 |
+
const uint START_ALIGN_K = 256;
|
| 339 |
+
// For Qi_K (block size 256), unroll whole 256 element tiles.
|
| 340 |
+
// For legacy quants (block size 32), unroll 8x.
|
| 341 |
+
const uint UNROLL_K = (QUANT_K == 256) ? 256 : (BK * 8);
|
| 342 |
+
const uint unroll_count = UNROLL_K / BK;
|
| 343 |
+
|
| 344 |
+
// Detect a fast path where all loads are entirely in bounds and no clamping is required
|
| 345 |
+
if ((ir + 1) * BM <= p.M && (ic + 1) * BN <= p.padded_N && (start_k % START_ALIGN_K) == 0 && (end_k % BK) == 0 &&
|
| 346 |
+
#if QUANT_K == 1
|
| 347 |
+
(stride_a % 8) == 0 &&
|
| 348 |
+
#endif
|
| 349 |
+
(stride_b % 8) == 0) {
|
| 350 |
+
// Hint to the compiler that values are aligned (want 16B alignment)
|
| 351 |
+
start_k &= ~(START_ALIGN_K-1);
|
| 352 |
+
stride_b &= ~7;
|
| 353 |
+
#if QUANT_K == 1
|
| 354 |
+
stride_a &= ~7;
|
| 355 |
+
#endif
|
| 356 |
+
|
| 357 |
+
tensorLayoutA = setTensorLayoutStrideNV(tensorLayoutA, stride_a, 1);
|
| 358 |
+
tensorLayoutB = setTensorLayoutStrideNV(tensorLayoutB, stride_b, 1);
|
| 359 |
+
|
| 360 |
+
uint k_iters = (end_k - start_k) / UNROLL_K;
|
| 361 |
+
uint block_k = start_k;
|
| 362 |
+
|
| 363 |
+
// fetch scale values for a tile of quants. These will be copied into shared memory.
|
| 364 |
+
// The fetches and stores are pipelined to hide the latency.
|
| 365 |
+
fetch_scales(ir * BM, pos_a, stride_a, start_k, tid, true);
|
| 366 |
+
|
| 367 |
+
if (enable_smaller_matrices && ic * BN + BNover4 >= p.N) {
|
| 368 |
+
coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BNover4, gl_MatrixUseAccumulator> sum = coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BNover4, gl_MatrixUseAccumulator>(0.0);
|
| 369 |
+
for (uint i = 0; i < k_iters; ++i) {
|
| 370 |
+
|
| 371 |
+
store_scales(tid);
|
| 372 |
+
if (block_k + UNROLL_K < end_k) {
|
| 373 |
+
fetch_scales(ir * BM, pos_a, stride_a, block_k + UNROLL_K, tid, true);
|
| 374 |
+
}
|
| 375 |
+
|
| 376 |
+
// Manually partial unroll
|
| 377 |
+
[[unroll]] for (uint j = 0; j < unroll_count; ++j) {
|
| 378 |
+
coopmat<MAT_TYPE, gl_ScopeWorkgroup, BM, BK, gl_MatrixUseA> mat_a;
|
| 379 |
+
coopmat<MAT_TYPE, gl_ScopeWorkgroup, BK, BNover4, gl_MatrixUseB> mat_b;
|
| 380 |
+
|
| 381 |
+
coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA);
|
| 382 |
+
coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, ic * BN, BNover4, block_k, BK), tensorViewTranspose);
|
| 383 |
+
|
| 384 |
+
sum = coopMatMulAdd(mat_a, mat_b, sum);
|
| 385 |
+
block_k += BK;
|
| 386 |
+
}
|
| 387 |
+
}
|
| 388 |
+
// Do any remaining iterations that were not unrolled
|
| 389 |
+
if (block_k < end_k) {
|
| 390 |
+
store_scales(tid);
|
| 391 |
+
}
|
| 392 |
+
while (block_k < end_k) {
|
| 393 |
+
coopmat<MAT_TYPE, gl_ScopeWorkgroup, BM, BK, gl_MatrixUseA> mat_a;
|
| 394 |
+
coopmat<MAT_TYPE, gl_ScopeWorkgroup, BK, BNover4, gl_MatrixUseB> mat_b;
|
| 395 |
+
|
| 396 |
+
coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA);
|
| 397 |
+
coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, ic * BN, BNover4, block_k, BK), tensorViewTranspose);
|
| 398 |
+
|
| 399 |
+
sum = coopMatMulAdd(mat_a, mat_b, sum);
|
| 400 |
+
block_k += BK;
|
| 401 |
+
}
|
| 402 |
+
#if defined(ACC_TYPE_MAX)
|
| 403 |
+
[[unroll]] for (uint i = 0; i < sum.length(); ++i) { sum[i] = clamp(sum[i], -ACC_TYPE_MAX, ACC_TYPE_MAX); }
|
| 404 |
+
#endif
|
| 405 |
+
|
| 406 |
+
coopmat<D_TYPE, gl_ScopeWorkgroup, BM, BNover4, gl_MatrixUseAccumulator> mat_d = coopmat<D_TYPE, gl_ScopeWorkgroup, BM, BNover4, gl_MatrixUseAccumulator>(sum);
|
| 407 |
+
|
| 408 |
+
coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BNover4, ir * BM, BM), tensorViewTranspose);
|
| 409 |
+
return;
|
| 410 |
+
} else if (enable_smaller_matrices && ic * BN + BNover2 >= p.N) {
|
| 411 |
+
coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BNover2, gl_MatrixUseAccumulator> sum = coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BNover2, gl_MatrixUseAccumulator>(0.0);
|
| 412 |
+
for (uint i = 0; i < k_iters; ++i) {
|
| 413 |
+
|
| 414 |
+
store_scales(tid);
|
| 415 |
+
if (block_k + UNROLL_K < end_k) {
|
| 416 |
+
fetch_scales(ir * BM, pos_a, stride_a, block_k + UNROLL_K, tid, true);
|
| 417 |
+
}
|
| 418 |
+
|
| 419 |
+
// Manually partial unroll
|
| 420 |
+
[[unroll]] for (uint j = 0; j < unroll_count; ++j) {
|
| 421 |
+
coopmat<MAT_TYPE, gl_ScopeWorkgroup, BM, BK, gl_MatrixUseA> mat_a;
|
| 422 |
+
coopmat<MAT_TYPE, gl_ScopeWorkgroup, BK, BNover2, gl_MatrixUseB> mat_b;
|
| 423 |
+
|
| 424 |
+
coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA);
|
| 425 |
+
coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, ic * BN, BNover2, block_k, BK), tensorViewTranspose);
|
| 426 |
+
|
| 427 |
+
sum = coopMatMulAdd(mat_a, mat_b, sum);
|
| 428 |
+
block_k += BK;
|
| 429 |
+
}
|
| 430 |
+
}
|
| 431 |
+
// Do any remaining iterations that were not unrolled
|
| 432 |
+
if (block_k < end_k) {
|
| 433 |
+
store_scales(tid);
|
| 434 |
+
}
|
| 435 |
+
while (block_k < end_k) {
|
| 436 |
+
coopmat<MAT_TYPE, gl_ScopeWorkgroup, BM, BK, gl_MatrixUseA> mat_a;
|
| 437 |
+
coopmat<MAT_TYPE, gl_ScopeWorkgroup, BK, BNover2, gl_MatrixUseB> mat_b;
|
| 438 |
+
|
| 439 |
+
coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA);
|
| 440 |
+
coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, ic * BN, BNover2, block_k, BK), tensorViewTranspose);
|
| 441 |
+
|
| 442 |
+
sum = coopMatMulAdd(mat_a, mat_b, sum);
|
| 443 |
+
block_k += BK;
|
| 444 |
+
}
|
| 445 |
+
#if defined(ACC_TYPE_MAX)
|
| 446 |
+
[[unroll]] for (uint i = 0; i < sum.length(); ++i) { sum[i] = clamp(sum[i], -ACC_TYPE_MAX, ACC_TYPE_MAX); }
|
| 447 |
+
#endif
|
| 448 |
+
|
| 449 |
+
coopmat<D_TYPE, gl_ScopeWorkgroup, BM, BNover2, gl_MatrixUseAccumulator> mat_d = coopmat<D_TYPE, gl_ScopeWorkgroup, BM, BNover2, gl_MatrixUseAccumulator>(sum);
|
| 450 |
+
|
| 451 |
+
coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BNover2, ir * BM, BM), tensorViewTranspose);
|
| 452 |
+
return;
|
| 453 |
+
} else {
|
| 454 |
+
coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BN, gl_MatrixUseAccumulator> sum = coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BN, gl_MatrixUseAccumulator>(0.0);
|
| 455 |
+
|
| 456 |
+
for (uint i = 0; i < k_iters; ++i) {
|
| 457 |
+
|
| 458 |
+
store_scales(tid);
|
| 459 |
+
if (block_k + UNROLL_K < end_k) {
|
| 460 |
+
fetch_scales(ir * BM, pos_a, stride_a, block_k + UNROLL_K, tid, true);
|
| 461 |
+
}
|
| 462 |
+
|
| 463 |
+
// Manually partial unroll
|
| 464 |
+
[[unroll]] for (uint j = 0; j < unroll_count; ++j) {
|
| 465 |
+
coopmat<MAT_TYPE, gl_ScopeWorkgroup, BM, BK, gl_MatrixUseA> mat_a;
|
| 466 |
+
coopmat<MAT_TYPE, gl_ScopeWorkgroup, BK, BN, gl_MatrixUseB> mat_b;
|
| 467 |
+
|
| 468 |
+
coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA);
|
| 469 |
+
coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, ic * BN, BN, block_k, BK), tensorViewTranspose);
|
| 470 |
+
|
| 471 |
+
sum = coopMatMulAdd(mat_a, mat_b, sum);
|
| 472 |
+
block_k += BK;
|
| 473 |
+
}
|
| 474 |
+
}
|
| 475 |
+
// Do any remaining iterations that were not unrolled
|
| 476 |
+
if (block_k < end_k) {
|
| 477 |
+
store_scales(tid);
|
| 478 |
+
}
|
| 479 |
+
while (block_k < end_k) {
|
| 480 |
+
coopmat<MAT_TYPE, gl_ScopeWorkgroup, BM, BK, gl_MatrixUseA> mat_a;
|
| 481 |
+
coopmat<MAT_TYPE, gl_ScopeWorkgroup, BK, BN, gl_MatrixUseB> mat_b;
|
| 482 |
+
|
| 483 |
+
coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA);
|
| 484 |
+
coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, ic * BN, BN, block_k, BK), tensorViewTranspose);
|
| 485 |
+
|
| 486 |
+
sum = coopMatMulAdd(mat_a, mat_b, sum);
|
| 487 |
+
block_k += BK;
|
| 488 |
+
}
|
| 489 |
+
#if defined(ACC_TYPE_MAX)
|
| 490 |
+
[[unroll]] for (uint i = 0; i < sum.length(); ++i) { sum[i] = clamp(sum[i], -ACC_TYPE_MAX, ACC_TYPE_MAX); }
|
| 491 |
+
#endif
|
| 492 |
+
|
| 493 |
+
coopmat<D_TYPE, gl_ScopeWorkgroup, BM, BN, gl_MatrixUseAccumulator> mat_d = coopmat<D_TYPE, gl_ScopeWorkgroup, BM, BN, gl_MatrixUseAccumulator>(sum);
|
| 494 |
+
|
| 495 |
+
coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BN, ir * BM, BM), tensorViewTranspose);
|
| 496 |
+
return;
|
| 497 |
+
}
|
| 498 |
+
} else
|
| 499 |
+
#endif // !defined(MUL_MAT_ID)
|
| 500 |
+
{
|
| 501 |
+
tensorLayoutA = setTensorLayoutStrideNV(tensorLayoutA, stride_a, 1);
|
| 502 |
+
|
| 503 |
+
tensorLayoutAClamp = setTensorLayoutStrideNV(tensorLayoutAClamp, stride_a, 1);
|
| 504 |
+
|
| 505 |
+
tensorLayoutB = setTensorLayoutStrideNV(tensorLayoutB, stride_b, 1);
|
| 506 |
+
|
| 507 |
+
tensorLayoutBClamp = setTensorLayoutStrideNV(tensorLayoutBClamp, stride_b, 1);
|
| 508 |
+
|
| 509 |
+
uint k_iters = (end_k - start_k + BK - 1) / BK;
|
| 510 |
+
|
| 511 |
+
fetch_scales(ir * BM, pos_a, stride_a, start_k, tid, false);
|
| 512 |
+
store_scales(tid);
|
| 513 |
+
|
| 514 |
+
#ifdef MUL_MAT_ID
|
| 515 |
+
if (enable_smaller_matrices && ic * BN + BNover4 >= _ne1) {
|
| 516 |
+
coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BNover4, gl_MatrixUseAccumulator> sum;
|
| 517 |
+
sum = coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BNover4, gl_MatrixUseAccumulator>(0.0);
|
| 518 |
+
|
| 519 |
+
[[dont_unroll]]
|
| 520 |
+
for (uint block_k = start_k, i = 0; i < k_iters; block_k += BK, ++i) {
|
| 521 |
+
|
| 522 |
+
if ((block_k % QUANT_K) == 0) {
|
| 523 |
+
store_scales(tid);
|
| 524 |
+
}
|
| 525 |
+
if (block_k + BK < end_k && ((block_k + BK) % QUANT_K) == 0) {
|
| 526 |
+
fetch_scales(ir * BM, pos_a, stride_a, block_k + BK, tid, false);
|
| 527 |
+
}
|
| 528 |
+
|
| 529 |
+
if ((ir + 1) * BM <= p.M && block_k + BK <= end_k) {
|
| 530 |
+
coopmat<MAT_TYPE, gl_ScopeWorkgroup, BM, BK, gl_MatrixUseA> mat_a;
|
| 531 |
+
coopmat<MAT_TYPE, gl_ScopeWorkgroup, BK, BNover4, gl_MatrixUseB> mat_b;
|
| 532 |
+
|
| 533 |
+
coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA);
|
| 534 |
+
coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, 0, BNover4, block_k, BK), tensorViewTranspose DECODEFUNCB);
|
| 535 |
+
|
| 536 |
+
sum = coopMatMulAdd(mat_a, mat_b, sum);
|
| 537 |
+
} else {
|
| 538 |
+
coopmat<MAT_TYPE, gl_ScopeWorkgroup, BM, BK, gl_MatrixUseA> mat_a;
|
| 539 |
+
coopmat<MAT_TYPE, gl_ScopeWorkgroup, BK, BNover4, gl_MatrixUseB> mat_b;
|
| 540 |
+
|
| 541 |
+
coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutAClamp, ir * BM, BM, block_k, BK) DECODEFUNCA);
|
| 542 |
+
coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, 0, BNover4, block_k, BK), tensorViewTranspose DECODEFUNCB);
|
| 543 |
+
|
| 544 |
+
sum = coopMatMulAdd(mat_a, mat_b, sum);
|
| 545 |
+
}
|
| 546 |
+
}
|
| 547 |
+
#if defined(ACC_TYPE_MAX)
|
| 548 |
+
[[unroll]] for (uint i = 0; i < sum.length(); ++i) { sum[i] = clamp(sum[i], -ACC_TYPE_MAX, ACC_TYPE_MAX); }
|
| 549 |
+
#endif
|
| 550 |
+
|
| 551 |
+
// Convert from ACC_TYPE to D_TYPE
|
| 552 |
+
coopmat<D_TYPE, gl_ScopeWorkgroup, BM, BNover4, gl_MatrixUseAccumulator> mat_d;
|
| 553 |
+
mat_d = coopmat<D_TYPE, gl_ScopeWorkgroup, BM, BNover4, gl_MatrixUseAccumulator>(sum);
|
| 554 |
+
|
| 555 |
+
// Call callback to store each element, remapping row through shared memory
|
| 556 |
+
coopMatPerElementNV(mat_d, mat_d, perElemOpD, ir, ic);
|
| 557 |
+
return;
|
| 558 |
+
}
|
| 559 |
+
if (enable_smaller_matrices && ic * BN + BNover2 >= _ne1) {
|
| 560 |
+
coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BNover2, gl_MatrixUseAccumulator> sum;
|
| 561 |
+
sum = coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BNover2, gl_MatrixUseAccumulator>(0.0);
|
| 562 |
+
|
| 563 |
+
[[dont_unroll]]
|
| 564 |
+
for (uint block_k = start_k, i = 0; i < k_iters; block_k += BK, ++i) {
|
| 565 |
+
|
| 566 |
+
if ((block_k % QUANT_K) == 0) {
|
| 567 |
+
store_scales(tid);
|
| 568 |
+
}
|
| 569 |
+
if (block_k + BK < end_k && ((block_k + BK) % QUANT_K) == 0) {
|
| 570 |
+
fetch_scales(ir * BM, pos_a, stride_a, block_k + BK, tid, false);
|
| 571 |
+
}
|
| 572 |
+
|
| 573 |
+
if ((ir + 1) * BM <= p.M && block_k + BK <= end_k) {
|
| 574 |
+
coopmat<MAT_TYPE, gl_ScopeWorkgroup, BM, BK, gl_MatrixUseA> mat_a;
|
| 575 |
+
coopmat<MAT_TYPE, gl_ScopeWorkgroup, BK, BNover2, gl_MatrixUseB> mat_b;
|
| 576 |
+
|
| 577 |
+
coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA);
|
| 578 |
+
coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, 0, BNover2, block_k, BK), tensorViewTranspose DECODEFUNCB);
|
| 579 |
+
|
| 580 |
+
sum = coopMatMulAdd(mat_a, mat_b, sum);
|
| 581 |
+
} else {
|
| 582 |
+
coopmat<MAT_TYPE, gl_ScopeWorkgroup, BM, BK, gl_MatrixUseA> mat_a;
|
| 583 |
+
coopmat<MAT_TYPE, gl_ScopeWorkgroup, BK, BNover2, gl_MatrixUseB> mat_b;
|
| 584 |
+
|
| 585 |
+
coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutAClamp, ir * BM, BM, block_k, BK) DECODEFUNCA);
|
| 586 |
+
coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, 0, BNover2, block_k, BK), tensorViewTranspose DECODEFUNCB);
|
| 587 |
+
|
| 588 |
+
sum = coopMatMulAdd(mat_a, mat_b, sum);
|
| 589 |
+
}
|
| 590 |
+
}
|
| 591 |
+
#if defined(ACC_TYPE_MAX)
|
| 592 |
+
[[unroll]] for (uint i = 0; i < sum.length(); ++i) { sum[i] = clamp(sum[i], -ACC_TYPE_MAX, ACC_TYPE_MAX); }
|
| 593 |
+
#endif
|
| 594 |
+
|
| 595 |
+
// Convert from ACC_TYPE to D_TYPE
|
| 596 |
+
coopmat<D_TYPE, gl_ScopeWorkgroup, BM, BNover2, gl_MatrixUseAccumulator> mat_d;
|
| 597 |
+
mat_d = coopmat<D_TYPE, gl_ScopeWorkgroup, BM, BNover2, gl_MatrixUseAccumulator>(sum);
|
| 598 |
+
|
| 599 |
+
// Call callback to store each element, remapping row through shared memory
|
| 600 |
+
coopMatPerElementNV(mat_d, mat_d, perElemOpD, ir, ic);
|
| 601 |
+
return;
|
| 602 |
+
}
|
| 603 |
+
#endif
|
| 604 |
+
coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BN, gl_MatrixUseAccumulator> sum;
|
| 605 |
+
sum = coopmat<ACC_TYPE, gl_ScopeWorkgroup, BM, BN, gl_MatrixUseAccumulator>(0.0);
|
| 606 |
+
|
| 607 |
+
[[dont_unroll]]
|
| 608 |
+
for (uint block_k = start_k, i = 0; i < k_iters; block_k += BK, ++i) {
|
| 609 |
+
|
| 610 |
+
if ((block_k % QUANT_K) == 0) {
|
| 611 |
+
store_scales(tid);
|
| 612 |
+
}
|
| 613 |
+
if (block_k + BK < end_k && ((block_k + BK) % QUANT_K) == 0) {
|
| 614 |
+
fetch_scales(ir * BM, pos_a, stride_a, block_k + BK, tid, false);
|
| 615 |
+
}
|
| 616 |
+
|
| 617 |
+
if ((ir + 1) * BM <= p.M && block_k + BK <= end_k) {
|
| 618 |
+
coopmat<MAT_TYPE, gl_ScopeWorkgroup, BM, BK, gl_MatrixUseA> mat_a;
|
| 619 |
+
coopmat<MAT_TYPE, gl_ScopeWorkgroup, BK, BN, gl_MatrixUseB> mat_b;
|
| 620 |
+
|
| 621 |
+
coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutA, ir * BM, BM, block_k, BK) DECODEFUNCA);
|
| 622 |
+
#ifdef MUL_MAT_ID
|
| 623 |
+
coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, 0, BN, block_k, BK), tensorViewTranspose DECODEFUNCB);
|
| 624 |
+
#else
|
| 625 |
+
coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutBClamp, ic * BN, BN, block_k, BK), tensorViewTranspose);
|
| 626 |
+
#endif
|
| 627 |
+
|
| 628 |
+
sum = coopMatMulAdd(mat_a, mat_b, sum);
|
| 629 |
+
} else {
|
| 630 |
+
coopmat<MAT_TYPE, gl_ScopeWorkgroup, BM, BK, gl_MatrixUseA> mat_a;
|
| 631 |
+
coopmat<MAT_TYPE, gl_ScopeWorkgroup, BK, BN, gl_MatrixUseB> mat_b;
|
| 632 |
+
|
| 633 |
+
coopMatLoadTensorNV(mat_a, data_a, pos_a, sliceTensorLayoutNV(tensorLayoutAClamp, ir * BM, BM, block_k, BK) DECODEFUNCA);
|
| 634 |
+
#ifdef MUL_MAT_ID
|
| 635 |
+
coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutB, 0, BN, block_k, BK), tensorViewTranspose DECODEFUNCB);
|
| 636 |
+
#else
|
| 637 |
+
coopMatLoadTensorNV(mat_b, data_b, pos_b, sliceTensorLayoutNV(tensorLayoutBClamp, ic * BN, BN, block_k, BK), tensorViewTranspose);
|
| 638 |
+
#endif
|
| 639 |
+
|
| 640 |
+
sum = coopMatMulAdd(mat_a, mat_b, sum);
|
| 641 |
+
}
|
| 642 |
+
}
|
| 643 |
+
#if defined(ACC_TYPE_MAX)
|
| 644 |
+
[[unroll]] for (uint i = 0; i < sum.length(); ++i) { sum[i] = clamp(sum[i], -ACC_TYPE_MAX, ACC_TYPE_MAX); }
|
| 645 |
+
#endif
|
| 646 |
+
|
| 647 |
+
// Convert from ACC_TYPE to D_TYPE
|
| 648 |
+
coopmat<D_TYPE, gl_ScopeWorkgroup, BM, BN, gl_MatrixUseAccumulator> mat_d;
|
| 649 |
+
mat_d = coopmat<D_TYPE, gl_ScopeWorkgroup, BM, BN, gl_MatrixUseAccumulator>(sum);
|
| 650 |
+
|
| 651 |
+
#ifdef MUL_MAT_ID
|
| 652 |
+
// Call callback to store each element, remapping row through shared memory
|
| 653 |
+
coopMatPerElementNV(mat_d, mat_d, perElemOpD, ir, ic);
|
| 654 |
+
#else
|
| 655 |
+
coopMatStoreTensorNV(mat_d, data_d, pos_d, sliceTensorLayoutNV(tensorLayoutD, ic * BN, BN, ir * BM, BM), tensorViewTranspose);
|
| 656 |
+
#endif
|
| 657 |
+
}
|
| 658 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_funcs.glsl
ADDED
|
@@ -0,0 +1,617 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
void load_a_to_shmem(const uint pos_a, const uint row, const uint col, const uint idx_m, const uint block, const uint end_k) {
|
| 2 |
+
#if defined(DATA_A_F32) || defined(DATA_A_F16)
|
| 3 |
+
#if LOAD_VEC_A == 8
|
| 4 |
+
if (ALIGNED != 0) {
|
| 5 |
+
const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
|
| 6 |
+
const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
|
| 7 |
+
FLOAT_TYPEV8 aa = FLOAT_TYPEV8(data_a[idx]);
|
| 8 |
+
buf_a[buf_idx ] = aa[0].xy;
|
| 9 |
+
buf_a[buf_idx + 1] = aa[0].zw;
|
| 10 |
+
buf_a[buf_idx + 2] = aa[1].xy;
|
| 11 |
+
buf_a[buf_idx + 3] = aa[1].zw;
|
| 12 |
+
return;
|
| 13 |
+
}
|
| 14 |
+
#elif LOAD_VEC_A == 4
|
| 15 |
+
if (ALIGNED != 0) {
|
| 16 |
+
const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
|
| 17 |
+
const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
|
| 18 |
+
FLOAT_TYPEV4 aa = FLOAT_TYPEV4(data_a[idx]);
|
| 19 |
+
buf_a[buf_idx ] = aa.xy;
|
| 20 |
+
buf_a[buf_idx + 1] = aa.zw;
|
| 21 |
+
return;
|
| 22 |
+
}
|
| 23 |
+
#endif
|
| 24 |
+
const uint idx = pos_a + col * p.stride_a + row * 2;
|
| 25 |
+
const uint buf_idx = col * SHMEM_STRIDE + row;
|
| 26 |
+
if (idx_m < p.M && block + row * 2 + 1 < end_k) {
|
| 27 |
+
buf_a[buf_idx] = FLOAT_TYPEV2(data_a_scalar[idx],
|
| 28 |
+
data_a_scalar[idx + 1]);
|
| 29 |
+
} else if (idx_m < p.M && block + row * 2 < end_k) {
|
| 30 |
+
buf_a[buf_idx] = FLOAT_TYPEV2(data_a_scalar[idx], 0.0f);
|
| 31 |
+
} else {
|
| 32 |
+
buf_a[buf_idx] = FLOAT_TYPEV2(0.0f);
|
| 33 |
+
}
|
| 34 |
+
#elif defined(DATA_A_BF16)
|
| 35 |
+
#if LOAD_VEC_A == 4
|
| 36 |
+
if (ALIGNED != 0) {
|
| 37 |
+
const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
|
| 38 |
+
const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
|
| 39 |
+
FLOAT_TYPEV4 aa = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_a[idx]));
|
| 40 |
+
buf_a[buf_idx ] = aa.xy;
|
| 41 |
+
buf_a[buf_idx + 1] = aa.zw;
|
| 42 |
+
return;
|
| 43 |
+
}
|
| 44 |
+
#endif
|
| 45 |
+
const uint idx = pos_a + col * p.stride_a + row * 2;
|
| 46 |
+
const uint buf_idx = col * SHMEM_STRIDE + row;
|
| 47 |
+
if (idx_m < p.M && block + row * 2 + 1 < end_k) {
|
| 48 |
+
buf_a[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_a_scalar[idx]),
|
| 49 |
+
TO_FLOAT_TYPE(data_a_scalar[idx + 1]));
|
| 50 |
+
} else if (idx_m < p.M && block + row * 2 < end_k) {
|
| 51 |
+
buf_a[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_a_scalar[idx]), 0.0f);
|
| 52 |
+
} else {
|
| 53 |
+
buf_a[buf_idx] = FLOAT_TYPEV2(0.0f);
|
| 54 |
+
}
|
| 55 |
+
#elif defined(DATA_A_Q4_0)
|
| 56 |
+
const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
|
| 57 |
+
const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 4;
|
| 58 |
+
|
| 59 |
+
const uint ib = idx / 4;
|
| 60 |
+
const uint iqs = idx & 0x03;
|
| 61 |
+
|
| 62 |
+
const float d = float(data_a_packed16[ib].d);
|
| 63 |
+
const uint vui = uint(data_a_packed16[ib].qs[2*iqs]) | (uint(data_a_packed16[ib].qs[2*iqs + 1]) << 16);
|
| 64 |
+
const vec4 v0 = (vec4(unpack8(vui & 0x0F0F0F0F)) - 8.0f) * d;
|
| 65 |
+
const vec4 v1 = (vec4(unpack8((vui >> 4) & 0x0F0F0F0F)) - 8.0f) * d;
|
| 66 |
+
|
| 67 |
+
buf_a[buf_idx ] = FLOAT_TYPEV2(v0.xy);
|
| 68 |
+
buf_a[buf_idx + 1] = FLOAT_TYPEV2(v0.zw);
|
| 69 |
+
buf_a[buf_idx + 8] = FLOAT_TYPEV2(v1.xy);
|
| 70 |
+
buf_a[buf_idx + 9] = FLOAT_TYPEV2(v1.zw);
|
| 71 |
+
#elif defined(DATA_A_Q4_1)
|
| 72 |
+
const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
|
| 73 |
+
const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 4;
|
| 74 |
+
|
| 75 |
+
const uint ib = idx / 4;
|
| 76 |
+
const uint iqs = idx & 0x03;
|
| 77 |
+
|
| 78 |
+
const vec2 dm = vec2(data_a_packed32[ib].dm);
|
| 79 |
+
const uint vui = data_a_packed32[ib].qs[iqs];
|
| 80 |
+
const vec4 v0 = vec4(unpack8(vui & 0x0F0F0F0F)) * dm.x + dm.y;
|
| 81 |
+
const vec4 v1 = vec4(unpack8((vui >> 4) & 0x0F0F0F0F)) * dm.x + dm.y;
|
| 82 |
+
|
| 83 |
+
buf_a[buf_idx ] = FLOAT_TYPEV2(v0.xy);
|
| 84 |
+
buf_a[buf_idx + 1 ] = FLOAT_TYPEV2(v0.zw);
|
| 85 |
+
buf_a[buf_idx + 8 ] = FLOAT_TYPEV2(v1.xy);
|
| 86 |
+
buf_a[buf_idx + 9 ] = FLOAT_TYPEV2(v1.zw);
|
| 87 |
+
#elif defined(DATA_A_Q5_0)
|
| 88 |
+
const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
|
| 89 |
+
const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 4;
|
| 90 |
+
|
| 91 |
+
const uint ib = idx / 8;
|
| 92 |
+
const uint iqs = idx & 0x07;
|
| 93 |
+
|
| 94 |
+
const float d = float(data_a_packed16[ib].d);
|
| 95 |
+
const uint uint_qh = uint(data_a_packed16[ib].qh[1]) << 16 | uint(data_a_packed16[ib].qh[0]);
|
| 96 |
+
const ivec2 qh0 = ivec2(((uint_qh >> 2*iqs) << 4) & 0x10, (uint_qh >> (2*iqs + 12)) & 0x10);
|
| 97 |
+
const ivec2 qh1 = ivec2(((uint_qh >> (2*iqs + 1)) << 4) & 0x10, (uint_qh >> (2*iqs + 13)) & 0x10);
|
| 98 |
+
|
| 99 |
+
const uint vui = uint(data_a_packed16[ib].qs[iqs]);
|
| 100 |
+
const vec4 v = (vec4((vui & 0xF) | qh0.x, ((vui >> 4) & 0xF) | qh0.y, ((vui >> 8) & 0xF) | qh1.x, (vui >> 12) | qh1.y) - 16.0f) * d;
|
| 101 |
+
|
| 102 |
+
buf_a[buf_idx ] = FLOAT_TYPEV2(v.xz);
|
| 103 |
+
buf_a[buf_idx + 8] = FLOAT_TYPEV2(v.yw);
|
| 104 |
+
#elif defined(DATA_A_Q5_1)
|
| 105 |
+
const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
|
| 106 |
+
const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 4;
|
| 107 |
+
|
| 108 |
+
const uint ib = idx / 4;
|
| 109 |
+
const uint iqs = idx & 0x03;
|
| 110 |
+
|
| 111 |
+
const vec2 dm = vec2(data_a_packed32[ib].dm);
|
| 112 |
+
const uint uint_qh = data_a_packed32[ib].qh;
|
| 113 |
+
const uvec2 qh0 = uvec2(((uint_qh >> 4*iqs) << 4) & 0x10, (uint_qh >> (4*iqs + 12)) & 0x10);
|
| 114 |
+
const uvec2 qh1 = uvec2(((uint_qh >> (4*iqs + 1)) << 4) & 0x10, (uint_qh >> (4*iqs + 13)) & 0x10);
|
| 115 |
+
const uvec2 qh2 = uvec2(((uint_qh >> (4*iqs + 2)) << 4) & 0x10, (uint_qh >> (4*iqs + 14)) & 0x10);
|
| 116 |
+
const uvec2 qh3 = uvec2(((uint_qh >> (4*iqs + 3)) << 4) & 0x10, (uint_qh >> (4*iqs + 15)) & 0x10);
|
| 117 |
+
|
| 118 |
+
const uint vui = data_a_packed32[ib].qs[iqs];
|
| 119 |
+
const vec4 v0 = vec4((vui & 0xF) | qh0.x, ((vui >> 4) & 0xF) | qh0.y, ((vui >> 8) & 0xF) | qh1.x, ((vui >> 12) & 0xF) | qh1.y) * dm.x + dm.y;
|
| 120 |
+
const vec4 v1 = vec4(((vui >> 16) & 0xF) | qh2.x, ((vui >> 20) & 0xF) | qh2.y, ((vui >> 24) & 0xF) | qh3.x, ((vui >> 28) & 0xF) | qh3.y) * dm.x + dm.y;
|
| 121 |
+
|
| 122 |
+
buf_a[buf_idx ] = FLOAT_TYPEV2(v0.xz);
|
| 123 |
+
buf_a[buf_idx + 1] = FLOAT_TYPEV2(v1.xz);
|
| 124 |
+
buf_a[buf_idx + 8] = FLOAT_TYPEV2(v0.yw);
|
| 125 |
+
buf_a[buf_idx + 9] = FLOAT_TYPEV2(v1.yw);
|
| 126 |
+
#elif defined(DATA_A_Q8_0)
|
| 127 |
+
const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
|
| 128 |
+
const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
|
| 129 |
+
|
| 130 |
+
const uint ib = idx / 8;
|
| 131 |
+
const uint iqs = idx & 0x07;
|
| 132 |
+
|
| 133 |
+
const float d = float(data_a_packed16[ib].d);
|
| 134 |
+
const i8vec2 v0 = unpack8(int32_t(data_a_packed16[ib].qs[2*iqs])).xy; // vec4 used due to #12147
|
| 135 |
+
const i8vec2 v1 = unpack8(int32_t(data_a_packed16[ib].qs[2*iqs + 1])).xy;
|
| 136 |
+
const vec4 v = vec4(v0.x, v0.y, v1.x, v1.y) * d;
|
| 137 |
+
|
| 138 |
+
buf_a[buf_idx ] = FLOAT_TYPEV2(v.xy);
|
| 139 |
+
buf_a[buf_idx + 1] = FLOAT_TYPEV2(v.zw);
|
| 140 |
+
#elif defined(DATA_A_Q1_0)
|
| 141 |
+
const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
|
| 142 |
+
const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
|
| 143 |
+
|
| 144 |
+
const uint ib = idx / 16;
|
| 145 |
+
const uint iqs = idx & 0xfu;
|
| 146 |
+
|
| 147 |
+
const float d = float(data_a[ib].d);
|
| 148 |
+
const uint bits = uint(data_a[ib].qs[iqs]);
|
| 149 |
+
|
| 150 |
+
buf_a[buf_idx ] = FLOAT_TYPEV2((bits & 0x01u) != 0u ? d : -d, (bits & 0x02u) != 0u ? d : -d);
|
| 151 |
+
buf_a[buf_idx + 1] = FLOAT_TYPEV2((bits & 0x04u) != 0u ? d : -d, (bits & 0x08u) != 0u ? d : -d);
|
| 152 |
+
buf_a[buf_idx + 2] = FLOAT_TYPEV2((bits & 0x10u) != 0u ? d : -d, (bits & 0x20u) != 0u ? d : -d);
|
| 153 |
+
buf_a[buf_idx + 3] = FLOAT_TYPEV2((bits & 0x40u) != 0u ? d : -d, (bits & 0x80u) != 0u ? d : -d);
|
| 154 |
+
#elif defined(DATA_A_Q2_K)
|
| 155 |
+
const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
|
| 156 |
+
const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
|
| 157 |
+
|
| 158 |
+
const uint ib = idx / 64; // 4 values per idx
|
| 159 |
+
const uint iqs = (idx % 64) * 2; // 0,2,4..126
|
| 160 |
+
|
| 161 |
+
const uint qsi = (iqs / 64) * 16 + (iqs % 16); // 0..15
|
| 162 |
+
const uint scalesi = iqs / 8; // 0..15
|
| 163 |
+
const uint qsshift = ((iqs % 64) / 16) * 2; // 0,2,4,6
|
| 164 |
+
|
| 165 |
+
const vec4 qs = vec4(unpack8((data_a_packed32[ib].qs[qsi / 2] >> qsshift) & 0x03030303));
|
| 166 |
+
const uint scales = data_a[ib].scales[scalesi];
|
| 167 |
+
const vec2 dm = vec2(data_a[ib].dm);
|
| 168 |
+
|
| 169 |
+
const vec4 v = dm.x * float(scales & 0xF) * qs - dm.y * float(scales >> 4);
|
| 170 |
+
|
| 171 |
+
buf_a[buf_idx ] = FLOAT_TYPEV2(v.xy);
|
| 172 |
+
buf_a[buf_idx + 1] = FLOAT_TYPEV2(v.zw);
|
| 173 |
+
#elif defined(DATA_A_Q3_K)
|
| 174 |
+
const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
|
| 175 |
+
const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
|
| 176 |
+
|
| 177 |
+
const uint ib = idx / 128; // 2 values per idx
|
| 178 |
+
const uint iqs = idx % 128; // 0..127
|
| 179 |
+
|
| 180 |
+
const uint n = iqs / 64; // 0,1
|
| 181 |
+
const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..62
|
| 182 |
+
const uint hmi = (iqs % 16) * 2; // 0,2,4..30
|
| 183 |
+
const uint j = (iqs % 64) / 4; // 0..3
|
| 184 |
+
const uint is = iqs / 8; // 0..15
|
| 185 |
+
const uint halfsplit = ((iqs % 64) / 16); // 0,1,2,3
|
| 186 |
+
const uint qsshift = halfsplit * 2; // 0,2,4,6
|
| 187 |
+
|
| 188 |
+
const int8_t us = int8_t(((data_a[ib].scales[is % 8] >> (4 * int(is / 8))) & 0xF)
|
| 189 |
+
| (((data_a[ib].scales[8 + (is % 4)] >> (2 * int(is / 4))) & 3) << 4));
|
| 190 |
+
const float dl = float(data_a[ib].d) * float(us - 32);
|
| 191 |
+
|
| 192 |
+
const vec2 qs = vec2(unpack8((uint(data_a_packed16[ib].qs[qsi / 2]) >> qsshift) & 0x0303).xy);
|
| 193 |
+
const vec2 hm = vec2(unpack8(((uint(data_a_packed16[ib].hmask[hmi / 2]) >> (4 * n + halfsplit)) & 0x0101 ^ 0x0101) << 2).xy);
|
| 194 |
+
|
| 195 |
+
buf_a[buf_idx] = FLOAT_TYPEV2(dl * (qs.x - hm.x),
|
| 196 |
+
dl * (qs.y - hm.y));
|
| 197 |
+
#elif defined(DATA_A_Q4_K)
|
| 198 |
+
const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
|
| 199 |
+
const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
|
| 200 |
+
|
| 201 |
+
const uint ib = idx / 64; // 4 values per idx
|
| 202 |
+
const uint iqs = (idx % 64) * 2; // 0,2,4..126
|
| 203 |
+
|
| 204 |
+
const uint n = iqs / 32; // 0,1,2,3
|
| 205 |
+
const uint b = (iqs % 32) / 16; // 0,1
|
| 206 |
+
const uint is = 2 * n + b; // 0..7
|
| 207 |
+
const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..126
|
| 208 |
+
|
| 209 |
+
const vec2 loadd = vec2(data_a[ib].dm);
|
| 210 |
+
|
| 211 |
+
const uvec3 scales = uvec3(data_a_packed32[ib].scales[0],
|
| 212 |
+
data_a_packed32[ib].scales[1],
|
| 213 |
+
data_a_packed32[ib].scales[2]);
|
| 214 |
+
const uint scalesoffs = (is & 3) * 8;
|
| 215 |
+
|
| 216 |
+
const uint scidx0 = (is < 4) ? 0 : 2;
|
| 217 |
+
const uint scidxshift0 = scalesoffs;
|
| 218 |
+
const uint scidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2;
|
| 219 |
+
const uint mbidx0 = (is < 4) ? 1 : 2;
|
| 220 |
+
const uint mbidxshift0 = (is < 4) ? scalesoffs : scalesoffs + 4;
|
| 221 |
+
const uint mbidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2;
|
| 222 |
+
|
| 223 |
+
const uint8_t sc = uint8_t(((scales[scidx0] >> scidxshift0) & 0xF) | ((scales[0] >> scidxshift1) & 0x30));
|
| 224 |
+
const uint8_t mbyte = uint8_t(((scales[mbidx0] >> mbidxshift0) & 0xF) | ((scales[1] >> mbidxshift1) & 0x30));
|
| 225 |
+
|
| 226 |
+
const float d = loadd.x * sc;
|
| 227 |
+
const float m = -loadd.y * mbyte;
|
| 228 |
+
|
| 229 |
+
const vec4 q = vec4(unpack8((data_a_packed32[ib].qs[qsi / 4] >> (b * 4)) & 0x0F0F0F0F));
|
| 230 |
+
|
| 231 |
+
buf_a[buf_idx ] = FLOAT_TYPEV2(fma(d, q.x, m), fma(d, q.y, m));
|
| 232 |
+
buf_a[buf_idx + 1] = FLOAT_TYPEV2(fma(d, q.z, m), fma(d, q.w, m));
|
| 233 |
+
#elif defined(DATA_A_Q5_K)
|
| 234 |
+
const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
|
| 235 |
+
const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
|
| 236 |
+
|
| 237 |
+
const uint ib = idx / 64; // 4 values per idx
|
| 238 |
+
const uint iqs = (idx % 64) * 2; // 0,2,4..126
|
| 239 |
+
|
| 240 |
+
const uint n = iqs / 32; // 0,1,2,3
|
| 241 |
+
const uint b = (iqs % 32) / 16; // 0,1
|
| 242 |
+
const uint is = 2 * n + b; // 0..7
|
| 243 |
+
const uint qsi = n * 32 + (iqs % 16) * 2; // 0,2,4..126
|
| 244 |
+
const uint qhi = (iqs % 16) * 2; // 0,2,4..30
|
| 245 |
+
|
| 246 |
+
const vec2 loadd = vec2(data_a[ib].dm);
|
| 247 |
+
|
| 248 |
+
const uvec3 scales = uvec3(data_a_packed32[ib].scales[0],
|
| 249 |
+
data_a_packed32[ib].scales[1],
|
| 250 |
+
data_a_packed32[ib].scales[2]);
|
| 251 |
+
const uint scalesoffs = (is & 3) * 8;
|
| 252 |
+
|
| 253 |
+
const uint scidx0 = (is < 4) ? 0 : 2;
|
| 254 |
+
const uint scidxshift0 = scalesoffs;
|
| 255 |
+
const uint scidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2;
|
| 256 |
+
const uint mbidx0 = (is < 4) ? 1 : 2;
|
| 257 |
+
const uint mbidxshift0 = (is < 4) ? scalesoffs : scalesoffs + 4;
|
| 258 |
+
const uint mbidxshift1 = (is < 4) ? scalesoffs : scalesoffs + 2;
|
| 259 |
+
|
| 260 |
+
const uint8_t sc = uint8_t(((scales[scidx0] >> scidxshift0) & 0xF) | ((scales[0] >> scidxshift1) & 0x30));
|
| 261 |
+
const uint8_t mbyte = uint8_t(((scales[mbidx0] >> mbidxshift0) & 0xF) | ((scales[1] >> mbidxshift1) & 0x30));
|
| 262 |
+
|
| 263 |
+
const float d = loadd.x * sc;
|
| 264 |
+
const float m = -loadd.y * mbyte;
|
| 265 |
+
|
| 266 |
+
const uint qs = (data_a_packed32[ib].qs[qsi / 4] >> (b * 4)) & 0x0F0F0F0F;
|
| 267 |
+
const uint qh = ((data_a_packed32[ib].qh[qhi / 4] >> (iqs / 16)) & 0x01010101) << 4;
|
| 268 |
+
const vec4 q = vec4(unpack8(qs | qh));
|
| 269 |
+
|
| 270 |
+
buf_a[buf_idx ] = FLOAT_TYPEV2(fma(d, q.x, m), fma(d, q.y, m));
|
| 271 |
+
buf_a[buf_idx + 1] = FLOAT_TYPEV2(fma(d, q.z, m), fma(d, q.w, m));
|
| 272 |
+
#elif defined(DATA_A_Q6_K)
|
| 273 |
+
const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
|
| 274 |
+
const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
|
| 275 |
+
|
| 276 |
+
const uint ib = idx / 128; // 2 values per idx
|
| 277 |
+
const uint iqs = idx % 128; // 0..127
|
| 278 |
+
|
| 279 |
+
const uint n = iqs / 64; // 0,1
|
| 280 |
+
const uint b = ((iqs % 64) / 32) * 4; // 0,4
|
| 281 |
+
const uint is_b = (iqs % 16) / 8; // 0,1
|
| 282 |
+
const uint qhshift = ((iqs % 64) / 16) * 2; // 0,2,4,6
|
| 283 |
+
const uint is = 8 * n + qhshift + is_b; // 0..15
|
| 284 |
+
const uint qsi = n * 32 + (iqs % 32); // 0..63
|
| 285 |
+
const uint qhi = n * 16 + (iqs % 16); // 0..31
|
| 286 |
+
|
| 287 |
+
const float dscale = float(data_a[ib].d) * float(data_a[ib].scales[is]);
|
| 288 |
+
|
| 289 |
+
const uint ql = (uint(data_a_packed16[ib].ql[qsi]) >> b) & 0x0F0F;
|
| 290 |
+
const uint qh = (uint(data_a_packed16[ib].qh[qhi]) >> qhshift) & 0x0303;
|
| 291 |
+
const vec2 q = (vec2(unpack8(ql | (qh << 4)).xy) - 32) * dscale;
|
| 292 |
+
|
| 293 |
+
buf_a[buf_idx] = FLOAT_TYPEV2(q.x, q.y);
|
| 294 |
+
#elif defined(DATA_A_IQ1_S)
|
| 295 |
+
const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
|
| 296 |
+
const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
|
| 297 |
+
|
| 298 |
+
const uint ib = idx / 32; // 8 values per idx
|
| 299 |
+
const uint ib32 = (idx % 32) / 4; // 0..7
|
| 300 |
+
const uint ib8 = idx % 32;
|
| 301 |
+
|
| 302 |
+
const float d = float(data_a[ib].d);
|
| 303 |
+
const uint qh = data_a[ib].qh[ib32];
|
| 304 |
+
const uint qs = data_a[ib].qs[ib8];
|
| 305 |
+
const float dl = d * (2 * bitfieldExtract(qh, 12, 3) + 1);
|
| 306 |
+
const float delta = ((qh & 0x8000) != 0) ? -IQ1S_DELTA : IQ1S_DELTA;
|
| 307 |
+
const int16_t grid = int16_t(iq1s_grid[qs | (bitfieldExtract(qh, 3 * int(ib8 & 3), 3) << 8)]);
|
| 308 |
+
|
| 309 |
+
[[unroll]] for (int k = 0; k < 4; ++k) {
|
| 310 |
+
buf_a[buf_idx + k] = FLOAT_TYPEV2(dl * (bitfieldExtract(grid, 4 * k , 2) + delta),
|
| 311 |
+
dl * (bitfieldExtract(grid, 4 * k + 2, 2) + delta));
|
| 312 |
+
}
|
| 313 |
+
#elif defined(DATA_A_IQ1_M)
|
| 314 |
+
const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
|
| 315 |
+
const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
|
| 316 |
+
|
| 317 |
+
const uint ib = idx / 32; // 8 values per idx
|
| 318 |
+
const uint ib8 = idx % 32;
|
| 319 |
+
const uint ib16 = ib8 / 2;
|
| 320 |
+
|
| 321 |
+
const uint16_t[4] scales = data_a[ib].scales;
|
| 322 |
+
const u16vec4 s = u16vec4(scales[0], scales[1], scales[2], scales[3]) >> 12;
|
| 323 |
+
const float d = float(unpackHalf2x16(s.x | (s.y << 4) | (s.z << 8) | (s.w << 12)).x);
|
| 324 |
+
const uint sc = scales[ib8 / 8];
|
| 325 |
+
const uint qs = data_a[ib].qs[ib8];
|
| 326 |
+
const uint qh = data_a[ib].qh[ib16] >> (4 * (ib8 & 1));
|
| 327 |
+
const float dl = d * (2 * bitfieldExtract(sc, 3 * int(ib16 & 3), 3) + 1);
|
| 328 |
+
const float delta = ((qh & 8) != 0) ? -IQ1M_DELTA : IQ1M_DELTA;
|
| 329 |
+
const int16_t grid = int16_t(iq1s_grid[qs | ((qh & 7) << 8)]);
|
| 330 |
+
|
| 331 |
+
[[unroll]] for (int k = 0; k < 4; ++k) {
|
| 332 |
+
buf_a[buf_idx + k] = FLOAT_TYPEV2(dl * (bitfieldExtract(grid, 4 * k , 2) + delta),
|
| 333 |
+
dl * (bitfieldExtract(grid, 4 * k + 2, 2) + delta));
|
| 334 |
+
}
|
| 335 |
+
#elif defined(DATA_A_IQ2_XXS)
|
| 336 |
+
const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
|
| 337 |
+
const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
|
| 338 |
+
|
| 339 |
+
const uint ib = idx / 32; // 8 values per idx
|
| 340 |
+
const uint ib32 = (idx % 32) / 4; // 0..7
|
| 341 |
+
const uint ib8 = idx % 4;
|
| 342 |
+
|
| 343 |
+
const float d = float(data_a[ib].d);
|
| 344 |
+
const uint qs = data_a[ib].qs[8 * ib32 + ib8];
|
| 345 |
+
const uint signs = pack32(u8vec4(
|
| 346 |
+
data_a[ib].qs[8*ib32 + 4],
|
| 347 |
+
data_a[ib].qs[8*ib32 + 5],
|
| 348 |
+
data_a[ib].qs[8*ib32 + 6],
|
| 349 |
+
data_a[ib].qs[8*ib32 + 7]
|
| 350 |
+
));
|
| 351 |
+
const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + (signs >> 28)));
|
| 352 |
+
const uint32_t sign7 = bitfieldExtract(signs, 7 * int(ib8), 7);
|
| 353 |
+
const uint sign = sign7 | (bitCount(sign7) << 7);
|
| 354 |
+
const uvec2 grid = iq2xxs_grid[qs];
|
| 355 |
+
const vec4 grid0 = vec4(unpack8(grid.x));
|
| 356 |
+
const vec4 grid1 = vec4(unpack8(grid.y));
|
| 357 |
+
|
| 358 |
+
buf_a[buf_idx ] = db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x,
|
| 359 |
+
(sign & 2) != 0 ? -grid0.y : grid0.y);
|
| 360 |
+
buf_a[buf_idx + 1] = db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z,
|
| 361 |
+
(sign & 8) != 0 ? -grid0.w : grid0.w);
|
| 362 |
+
buf_a[buf_idx + 2] = db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x,
|
| 363 |
+
(sign & 32) != 0 ? -grid1.y : grid1.y);
|
| 364 |
+
buf_a[buf_idx + 3] = db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z,
|
| 365 |
+
(sign & 128) != 0 ? -grid1.w : grid1.w);
|
| 366 |
+
#elif defined(DATA_A_IQ2_XS)
|
| 367 |
+
const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
|
| 368 |
+
const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
|
| 369 |
+
|
| 370 |
+
const uint ib = idx / 32; // 8 values per idx
|
| 371 |
+
const uint ib32 = (idx % 32) / 4; // 0..7
|
| 372 |
+
const uint ib8 = idx % 4; // 0..3
|
| 373 |
+
|
| 374 |
+
const float d = float(data_a[ib].d);
|
| 375 |
+
const uint scale = (data_a[ib].scales[ib32] >> (2 * (ib8 & 2))) & 0xf;
|
| 376 |
+
const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + scale));
|
| 377 |
+
const uint qs = data_a[ib].qs[4 * ib32 + ib8];
|
| 378 |
+
const uint sign7 = qs >> 9;
|
| 379 |
+
const uint sign = sign7 | (bitCount(sign7) << 7);
|
| 380 |
+
const uvec2 grid = iq2xs_grid[qs & 511];
|
| 381 |
+
const vec4 grid0 = vec4(unpack8(grid.x));
|
| 382 |
+
const vec4 grid1 = vec4(unpack8(grid.y));
|
| 383 |
+
|
| 384 |
+
buf_a[buf_idx ] = db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x,
|
| 385 |
+
(sign & 2) != 0 ? -grid0.y : grid0.y);
|
| 386 |
+
buf_a[buf_idx + 1] = db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z,
|
| 387 |
+
(sign & 8) != 0 ? -grid0.w : grid0.w);
|
| 388 |
+
buf_a[buf_idx + 2] = db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x,
|
| 389 |
+
(sign & 32) != 0 ? -grid1.y : grid1.y);
|
| 390 |
+
buf_a[buf_idx + 3] = db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z,
|
| 391 |
+
(sign & 128) != 0 ? -grid1.w : grid1.w);
|
| 392 |
+
#elif defined(DATA_A_IQ2_S)
|
| 393 |
+
const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
|
| 394 |
+
const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
|
| 395 |
+
|
| 396 |
+
const uint ib = idx / 32; // 8 values per idx
|
| 397 |
+
const uint ib8 = idx % 32; // 0..31
|
| 398 |
+
const uint ib32 = ib8 / 4; // 0..7
|
| 399 |
+
|
| 400 |
+
const uint scale = (data_a[ib].scales[ib32] >> (2 * (ib8 & 2))) & 0xf;
|
| 401 |
+
const uint qs = data_a[ib].qs[ib8];
|
| 402 |
+
const uint qh = data_a[ib].qh[ib32];
|
| 403 |
+
const uint qhshift = 2 * (ib8 % 4);
|
| 404 |
+
const uint sign = data_a[ib].qs[QUANT_K / 8 + ib8];
|
| 405 |
+
|
| 406 |
+
const float d = float(data_a[ib].d);
|
| 407 |
+
const FLOAT_TYPE db = FLOAT_TYPE(d * 0.25 * (0.5 + scale));
|
| 408 |
+
const uvec2 grid = iq2s_grid[qs | ((qh << (8 - qhshift)) & 0x300)];
|
| 409 |
+
const vec4 grid0 = vec4(unpack8(grid.x));
|
| 410 |
+
const vec4 grid1 = vec4(unpack8(grid.y));
|
| 411 |
+
|
| 412 |
+
buf_a[buf_idx ] = db * FLOAT_TYPEV2((sign & 1) != 0 ? -grid0.x : grid0.x,
|
| 413 |
+
(sign & 2) != 0 ? -grid0.y : grid0.y);
|
| 414 |
+
buf_a[buf_idx + 1] = db * FLOAT_TYPEV2((sign & 4) != 0 ? -grid0.z : grid0.z,
|
| 415 |
+
(sign & 8) != 0 ? -grid0.w : grid0.w);
|
| 416 |
+
buf_a[buf_idx + 2] = db * FLOAT_TYPEV2((sign & 16) != 0 ? -grid1.x : grid1.x,
|
| 417 |
+
(sign & 32) != 0 ? -grid1.y : grid1.y);
|
| 418 |
+
buf_a[buf_idx + 3] = db * FLOAT_TYPEV2((sign & 64) != 0 ? -grid1.z : grid1.z,
|
| 419 |
+
(sign & 128) != 0 ? -grid1.w : grid1.w);
|
| 420 |
+
#elif defined(DATA_A_IQ3_XXS)
|
| 421 |
+
const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
|
| 422 |
+
const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
|
| 423 |
+
|
| 424 |
+
const uint ib = idx / 64; // 4 values per idx
|
| 425 |
+
const uint iqs = idx % 64; // 0..63
|
| 426 |
+
const uint is = QUANT_K / 4 + 4 * (iqs / 8); // 8 values
|
| 427 |
+
|
| 428 |
+
const float d = float(data_a[ib].d);
|
| 429 |
+
const uint qs = data_a[ib].qs[iqs];
|
| 430 |
+
const uint signs = pack32(u16vec2(
|
| 431 |
+
data_a_packed16[ib].qs[is/2],
|
| 432 |
+
data_a_packed16[ib].qs[is/2+1]
|
| 433 |
+
));
|
| 434 |
+
const float db = d * 0.5 * (0.5 + (signs >> 28));
|
| 435 |
+
const uint32_t sign7 = bitfieldExtract(signs, 7 * (int(iqs / 2) % 4), 7);
|
| 436 |
+
const uint sign = (sign7 | (bitCount(sign7) << 7)) >> (4 * (idx % 2));
|
| 437 |
+
const uint grid = iq3xxs_grid[qs];
|
| 438 |
+
const vec4 v = db * vec4(unpack8(grid));
|
| 439 |
+
|
| 440 |
+
buf_a[buf_idx ] = FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x,
|
| 441 |
+
(sign & 2) != 0 ? -v.y : v.y);
|
| 442 |
+
buf_a[buf_idx + 1] = FLOAT_TYPEV2((sign & 4) != 0 ? -v.z : v.z,
|
| 443 |
+
(sign & 8) != 0 ? -v.w : v.w);
|
| 444 |
+
#elif defined(DATA_A_IQ3_S)
|
| 445 |
+
const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
|
| 446 |
+
const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
|
| 447 |
+
|
| 448 |
+
const uint ib = idx / 64; // 4 values per idx
|
| 449 |
+
const uint iqs = idx % 64; // 0..63
|
| 450 |
+
const uint iqh = iqs / 8;
|
| 451 |
+
|
| 452 |
+
const float d = float(data_a[ib].d);
|
| 453 |
+
const uint qs = data_a[ib].qs[iqs];
|
| 454 |
+
const uint qh = data_a[ib].qh[iqh];
|
| 455 |
+
const int8_t sign = int8_t(data_a[ib].signs[iqs / 2] >> (4 * (idx % 2)));
|
| 456 |
+
const uint scale = data_a[ib].scales[iqs / 16];
|
| 457 |
+
const i8vec2 sign01 = i8vec2(1 - (2 & i8vec2(sign << 1, sign)));
|
| 458 |
+
const float db = d * (1 + 2 * ((scale >> (4 * (iqh & 1))) & 0xf));
|
| 459 |
+
const uint32_t grid = iq3s_grid[qs | ((qh << (8 - (iqs % 8))) & 256)];
|
| 460 |
+
const vec4 v = db * vec4(unpack8(grid));
|
| 461 |
+
|
| 462 |
+
buf_a[buf_idx ] = FLOAT_TYPEV2((sign & 1) != 0 ? -v.x : v.x,
|
| 463 |
+
(sign & 2) != 0 ? -v.y : v.y);
|
| 464 |
+
buf_a[buf_idx + 1] = FLOAT_TYPEV2((sign & 4) != 0 ? -v.z : v.z,
|
| 465 |
+
(sign & 8) != 0 ? -v.w : v.w);
|
| 466 |
+
#elif defined(DATA_A_IQ4_XS)
|
| 467 |
+
const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
|
| 468 |
+
const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 2;
|
| 469 |
+
|
| 470 |
+
const uint ib = idx / 64; // 4 values per idx
|
| 471 |
+
const uint ib32 = (idx % 64) / 8; // 0..7
|
| 472 |
+
const uint iq = 4 * ib32 + (idx % 4);
|
| 473 |
+
|
| 474 |
+
const uint sl = (data_a[ib].scales_l[ib32/2] >> (4 * (ib32 & 1))) & 0xF;
|
| 475 |
+
const uint sh = ((data_a[ib].scales_h) >> (2 * ib32)) & 3;
|
| 476 |
+
const uint qshift = idx & 4;
|
| 477 |
+
u8vec4 qs = unpack8((uint(data_a_packed32[ib].qs[iq]) >> qshift) & 0x0F0F0F0F);
|
| 478 |
+
|
| 479 |
+
const float d = float(data_a[ib].d);
|
| 480 |
+
const vec4 v = d * float(int(sl | (sh << 4)) - 32) * vec4(kvalues_iq4nl[qs.x], kvalues_iq4nl[qs.y], kvalues_iq4nl[qs.z], kvalues_iq4nl[qs.w]);
|
| 481 |
+
|
| 482 |
+
buf_a[buf_idx ] = FLOAT_TYPEV2(v.xy);
|
| 483 |
+
buf_a[buf_idx + 1] = FLOAT_TYPEV2(v.zw);
|
| 484 |
+
#elif defined(DATA_A_IQ4_NL)
|
| 485 |
+
const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
|
| 486 |
+
const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 4;
|
| 487 |
+
|
| 488 |
+
const uint ib = idx / 8;
|
| 489 |
+
const uint iqs = idx & 0x07;
|
| 490 |
+
|
| 491 |
+
const FLOAT_TYPE d = FLOAT_TYPE(data_a_packed16[ib].d);
|
| 492 |
+
const uint vui = uint(data_a_packed16[ib].qs[iqs]);
|
| 493 |
+
|
| 494 |
+
buf_a[buf_idx ] = d * FLOAT_TYPEV2(kvalues_iq4nl[vui & 0xF],
|
| 495 |
+
kvalues_iq4nl[bitfieldExtract(vui, 8, 4)]);
|
| 496 |
+
buf_a[buf_idx + 8] = d * FLOAT_TYPEV2(kvalues_iq4nl[bitfieldExtract(vui, 4, 4)],
|
| 497 |
+
kvalues_iq4nl[vui >> 12]);
|
| 498 |
+
#elif defined(DATA_A_MXFP4)
|
| 499 |
+
const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
|
| 500 |
+
const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_A / 4;
|
| 501 |
+
|
| 502 |
+
const uint ib = idx / 8;
|
| 503 |
+
const uint iqs = (idx & 0x07) * 2;
|
| 504 |
+
|
| 505 |
+
const float d = e8m0_to_fp32(data_a[ib].e) * 0.5;
|
| 506 |
+
const uint vui = uint(data_a[ib].qs[iqs]);
|
| 507 |
+
const uint vui2 = uint(data_a[ib].qs[iqs+1]);
|
| 508 |
+
|
| 509 |
+
buf_a[buf_idx ] = FLOAT_TYPEV2(kvalues_mxfp4[vui & 0xF] * d,
|
| 510 |
+
kvalues_mxfp4[vui2 & 0xF] * d);
|
| 511 |
+
buf_a[buf_idx + 8] = FLOAT_TYPEV2(kvalues_mxfp4[vui >> 4] * d,
|
| 512 |
+
kvalues_mxfp4[vui2 >> 4] * d);
|
| 513 |
+
#elif defined(DATA_A_NVFP4)
|
| 514 |
+
const uint idx = pos_a + col * p.stride_a / LOAD_VEC_A + row;
|
| 515 |
+
// lo and hi nibbles are 8 elements apart, which doesn't quite line up with
|
| 516 |
+
// how the thread mapping and buf_idx calculation works for other types.
|
| 517 |
+
const uint buf_idx = col * SHMEM_STRIDE + (row & 3) + (row & ~3) * 2;
|
| 518 |
+
|
| 519 |
+
const uint ib = idx / 16u;
|
| 520 |
+
const uint sub = (idx & 0xC) >> 2;
|
| 521 |
+
const uint iqs = (idx & 0xF) * 2;
|
| 522 |
+
const float d = ue4m3_to_fp32(data_a[ib].d[sub]) * 0.5;
|
| 523 |
+
const uint vui = uint(data_a[ib].qs[iqs]);
|
| 524 |
+
const uint vui2 = uint(data_a[ib].qs[iqs+1]);
|
| 525 |
+
|
| 526 |
+
buf_a[buf_idx ] = FLOAT_TYPEV2(kvalues_mxfp4[vui & 0xF] * d,
|
| 527 |
+
kvalues_mxfp4[vui2 & 0xF] * d);
|
| 528 |
+
buf_a[buf_idx + 4] = FLOAT_TYPEV2(kvalues_mxfp4[vui >> 4] * d,
|
| 529 |
+
kvalues_mxfp4[vui2 >> 4] * d);
|
| 530 |
+
#endif
|
| 531 |
+
}
|
| 532 |
+
|
| 533 |
+
#if !defined(MUL_MAT_ID)
|
| 534 |
+
void load_b_to_shmem(const uint pos_b, const uint row, const uint col, const uint idx_n, const uint block, const uint end_k) {
|
| 535 |
+
#if LOAD_VEC_B == 8
|
| 536 |
+
if (ALIGNED != 0) {
|
| 537 |
+
// Not supported for b_type bf16 because bf16mat2x4 does not exist
|
| 538 |
+
const uint idx = pos_b + col * p.stride_b / LOAD_VEC_B + row;
|
| 539 |
+
const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2;
|
| 540 |
+
FLOAT_TYPEV8 bb = FLOAT_TYPEV8(data_b[idx]);
|
| 541 |
+
buf_b[buf_idx + 0] = bb[0].xy;
|
| 542 |
+
buf_b[buf_idx + 1] = bb[0].zw;
|
| 543 |
+
buf_b[buf_idx + 2] = bb[1].xy;
|
| 544 |
+
buf_b[buf_idx + 3] = bb[1].zw;
|
| 545 |
+
return;
|
| 546 |
+
}
|
| 547 |
+
#elif LOAD_VEC_B == 4
|
| 548 |
+
if (ALIGNED != 0) {
|
| 549 |
+
const uint idx = pos_b + col * p.stride_b / LOAD_VEC_B + row;
|
| 550 |
+
const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2;
|
| 551 |
+
#if defined(DATA_B_BF16)
|
| 552 |
+
FLOAT_TYPEV4 bb = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_b[idx]));
|
| 553 |
+
#else
|
| 554 |
+
FLOAT_TYPEV4 bb = FLOAT_TYPEV4(data_b[idx]);
|
| 555 |
+
#endif
|
| 556 |
+
buf_b[buf_idx + 0] = bb.xy;
|
| 557 |
+
buf_b[buf_idx + 1] = bb.zw;
|
| 558 |
+
return;
|
| 559 |
+
}
|
| 560 |
+
#endif
|
| 561 |
+
const uint idx = pos_b + col * p.stride_b + row * 2;
|
| 562 |
+
const uint buf_idx = col * SHMEM_STRIDE + row;
|
| 563 |
+
if (idx_n < p.N && block + row * 2 + 1 < end_k) {
|
| 564 |
+
buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]),
|
| 565 |
+
TO_FLOAT_TYPE(data_b_scalar[idx + 1]));
|
| 566 |
+
} else if (idx_n < p.N && block + row * 2 < end_k) {
|
| 567 |
+
buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), 0.0f);
|
| 568 |
+
} else {
|
| 569 |
+
buf_b[buf_idx] = FLOAT_TYPEV2(0.0f);
|
| 570 |
+
}
|
| 571 |
+
}
|
| 572 |
+
#else
|
| 573 |
+
void load_b_to_shmem(const uint pos_b, const uint row, const uint col, const uint ic, const uint _ne1, const uint block, const uint end_k) {
|
| 574 |
+
#if LOAD_VEC_B == 8
|
| 575 |
+
if (ALIGNED != 0) {
|
| 576 |
+
// Not supported for b_type bf16 because bf16mat2x4 does not exist
|
| 577 |
+
const u16vec2 row_idx = row_ids[col];
|
| 578 |
+
const uint idx = pos_b + row_idx.y * p.batch_stride_b / LOAD_VEC_B + (row_idx.x % p.ne11) * p.stride_b / LOAD_VEC_B + row;
|
| 579 |
+
const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2;
|
| 580 |
+
FLOAT_TYPEV8 bb = FLOAT_TYPEV8(data_b[idx]);
|
| 581 |
+
buf_b[buf_idx + 0] = bb[0].xy;
|
| 582 |
+
buf_b[buf_idx + 1] = bb[0].zw;
|
| 583 |
+
buf_b[buf_idx + 2] = bb[1].xy;
|
| 584 |
+
buf_b[buf_idx + 3] = bb[1].zw;
|
| 585 |
+
return;
|
| 586 |
+
}
|
| 587 |
+
#elif LOAD_VEC_B == 4
|
| 588 |
+
if (ALIGNED != 0) {
|
| 589 |
+
const u16vec2 row_idx = row_ids[col];
|
| 590 |
+
const uint idx = pos_b + row_idx.y * p.batch_stride_b / LOAD_VEC_B + (row_idx.x % p.ne11) * p.stride_b / LOAD_VEC_B + row;
|
| 591 |
+
const uint buf_idx = col * SHMEM_STRIDE + row * LOAD_VEC_B / 2;
|
| 592 |
+
#if defined(DATA_B_BF16)
|
| 593 |
+
FLOAT_TYPEV4 bb = FLOAT_TYPEV4(TO_FLOAT_TYPE(data_b[idx]));
|
| 594 |
+
#else
|
| 595 |
+
FLOAT_TYPEV4 bb = FLOAT_TYPEV4(data_b[idx]);
|
| 596 |
+
#endif
|
| 597 |
+
buf_b[buf_idx + 0] = bb.xy;
|
| 598 |
+
buf_b[buf_idx + 1] = bb.zw;
|
| 599 |
+
return;
|
| 600 |
+
}
|
| 601 |
+
#endif
|
| 602 |
+
const uint row_i = ic * BN + col;
|
| 603 |
+
const uint buf_idx = col * SHMEM_STRIDE + row;
|
| 604 |
+
if (row_i < _ne1 && block + row * 2 + 1 < end_k) {
|
| 605 |
+
const u16vec2 row_idx = row_ids[col];
|
| 606 |
+
const uint idx = pos_b + row_idx.y * p.batch_stride_b + (row_idx.x % p.ne11) * p.stride_b + row * 2;
|
| 607 |
+
buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]),
|
| 608 |
+
TO_FLOAT_TYPE(data_b_scalar[idx + 1]));
|
| 609 |
+
} else if (row_i < _ne1 && block + row * 2 < end_k) {
|
| 610 |
+
const u16vec2 row_idx = row_ids[col];
|
| 611 |
+
const uint idx = pos_b + row_idx.y * p.batch_stride_b + (row_idx.x % p.ne11) * p.stride_b + row * 2;
|
| 612 |
+
buf_b[buf_idx] = FLOAT_TYPEV2(TO_FLOAT_TYPE(data_b_scalar[idx]), 0.0f);
|
| 613 |
+
} else {
|
| 614 |
+
buf_b[buf_idx] = FLOAT_TYPEV2(0.0f);
|
| 615 |
+
}
|
| 616 |
+
}
|
| 617 |
+
#endif
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mm_id_funcs.glsl
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#ifdef MUL_MAT_ID
|
| 2 |
+
shared u16vec2 row_ids[BN];
|
| 3 |
+
uint _ne1;
|
| 4 |
+
|
| 5 |
+
#ifdef MUL_MAT_ID_USE_SUBGROUPS
|
| 6 |
+
shared uvec4 ballots_sh[NUM_WARPS];
|
| 7 |
+
|
| 8 |
+
void load_row_ids(uint expert_idx, bool nei0_is_pow2, uint ic) {
|
| 9 |
+
_ne1 = 0;
|
| 10 |
+
uint num_elements = p.nei1 * p.nei0;
|
| 11 |
+
uint nei0shift = findLSB(p.nei0);
|
| 12 |
+
|
| 13 |
+
uint ids[16];
|
| 14 |
+
uint iter = 0;
|
| 15 |
+
|
| 16 |
+
uint expert_count = data_expert_count[expert_idx];
|
| 17 |
+
|
| 18 |
+
for (uint j = 0; j < num_elements; j += BLOCK_SIZE) {
|
| 19 |
+
// prefetch up to 16 elements
|
| 20 |
+
if (iter == 0) {
|
| 21 |
+
[[unroll]] for (uint k = 0; k < 16; ++k) {
|
| 22 |
+
uint i = j + gl_LocalInvocationIndex + k*BLOCK_SIZE;
|
| 23 |
+
bool in_range = i < num_elements;
|
| 24 |
+
uint ii1;
|
| 25 |
+
if (nei0_is_pow2) {
|
| 26 |
+
ii1 = i >> nei0shift;
|
| 27 |
+
} else {
|
| 28 |
+
ii1 = i / p.nei0;
|
| 29 |
+
}
|
| 30 |
+
uint ii0 = i - ii1 * p.nei0;
|
| 31 |
+
ids[k] = in_range ? data_ids[ii1*p.nbi1 + ii0] : 0;
|
| 32 |
+
}
|
| 33 |
+
}
|
| 34 |
+
uint i = j + gl_LocalInvocationIndex;
|
| 35 |
+
bool in_range = i < num_elements;
|
| 36 |
+
uint ii1;
|
| 37 |
+
if (nei0_is_pow2) {
|
| 38 |
+
ii1 = i >> nei0shift;
|
| 39 |
+
} else {
|
| 40 |
+
ii1 = i / p.nei0;
|
| 41 |
+
}
|
| 42 |
+
uint ii0 = i - ii1 * p.nei0;
|
| 43 |
+
uint id = ids[iter++];
|
| 44 |
+
uvec4 ballot = subgroupBallot(in_range && id == expert_idx);
|
| 45 |
+
|
| 46 |
+
if (gl_SubgroupInvocationID == 0) {
|
| 47 |
+
ballots_sh[gl_SubgroupID] = ballot;
|
| 48 |
+
}
|
| 49 |
+
barrier();
|
| 50 |
+
|
| 51 |
+
uint subgroup_base = 0;
|
| 52 |
+
uint total = 0;
|
| 53 |
+
for (uint k = 0; k < gl_NumSubgroups; ++k) {
|
| 54 |
+
if (k == gl_SubgroupID) {
|
| 55 |
+
subgroup_base = total;
|
| 56 |
+
}
|
| 57 |
+
total += subgroupBallotBitCount(ballots_sh[k]);
|
| 58 |
+
}
|
| 59 |
+
barrier();
|
| 60 |
+
|
| 61 |
+
uint idx = subgroup_base + subgroupBallotExclusiveBitCount(ballot);
|
| 62 |
+
if (in_range && id == expert_idx && _ne1 + idx >= ic * BN && _ne1 + idx < (ic + 1) * BN) {
|
| 63 |
+
row_ids[_ne1 + idx - ic * BN] = u16vec2(ii0, ii1);
|
| 64 |
+
}
|
| 65 |
+
_ne1 += total;
|
| 66 |
+
iter &= 15;
|
| 67 |
+
if (_ne1 >= (ic + 1) * BN || _ne1 == expert_count) {
|
| 68 |
+
break;
|
| 69 |
+
}
|
| 70 |
+
}
|
| 71 |
+
barrier();
|
| 72 |
+
}
|
| 73 |
+
#endif // MUL_MAT_ID_USE_SUBGROUPS
|
| 74 |
+
#endif // MUL_MAT_ID
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp
ADDED
|
@@ -0,0 +1,311 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
|
| 3 |
+
#extension GL_EXT_control_flow_attributes : enable
|
| 4 |
+
#extension GL_EXT_shader_16bit_storage : require
|
| 5 |
+
#extension GL_EXT_shader_explicit_arithmetic_types_int8 : require
|
| 6 |
+
|
| 7 |
+
#extension GL_EXT_integer_dot_product : require
|
| 8 |
+
|
| 9 |
+
#ifdef FLOAT16
|
| 10 |
+
#extension GL_EXT_shader_explicit_arithmetic_types_float16 : require
|
| 11 |
+
#endif
|
| 12 |
+
|
| 13 |
+
#if defined(MUL_MAT_ID_USE_SUBGROUPS)
|
| 14 |
+
#extension GL_KHR_shader_subgroup_basic : enable
|
| 15 |
+
#extension GL_KHR_shader_subgroup_ballot : enable
|
| 16 |
+
#endif
|
| 17 |
+
|
| 18 |
+
#ifdef MUL_MAT_ID
|
| 19 |
+
#extension GL_EXT_shader_explicit_arithmetic_types_int16 : require
|
| 20 |
+
#endif
|
| 21 |
+
|
| 22 |
+
#include "types.glsl"
|
| 23 |
+
|
| 24 |
+
layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
|
| 25 |
+
|
| 26 |
+
layout (binding = 0) readonly buffer A {A_TYPE data_a[];};
|
| 27 |
+
#if defined(A_TYPE_PACKED16)
|
| 28 |
+
layout (binding = 0) readonly buffer A_PACKED16 {A_TYPE_PACKED16 data_a_packed16[];};
|
| 29 |
+
#endif
|
| 30 |
+
#if defined(A_TYPE_PACKED32)
|
| 31 |
+
layout (binding = 0) readonly buffer A_PACKED32 {A_TYPE_PACKED32 data_a_packed32[];};
|
| 32 |
+
#endif
|
| 33 |
+
layout (binding = 1) readonly buffer B {block_q8_1_x4_packed128 data_b[];};
|
| 34 |
+
layout (binding = 2) writeonly buffer D {D_TYPE data_d[];};
|
| 35 |
+
|
| 36 |
+
#ifdef MUL_MAT_ID
|
| 37 |
+
layout (binding = 3) readonly buffer IDS {int data_ids[];};
|
| 38 |
+
layout (binding = 4) readonly buffer Counts {int data_expert_count[];};
|
| 39 |
+
#endif
|
| 40 |
+
|
| 41 |
+
layout (push_constant) uniform parameter
|
| 42 |
+
{
|
| 43 |
+
uint M;
|
| 44 |
+
uint N;
|
| 45 |
+
uint K;
|
| 46 |
+
uint stride_a;
|
| 47 |
+
uint stride_b;
|
| 48 |
+
uint stride_d;
|
| 49 |
+
|
| 50 |
+
uint batch_stride_a;
|
| 51 |
+
uint batch_stride_b;
|
| 52 |
+
uint batch_stride_d;
|
| 53 |
+
|
| 54 |
+
#ifdef MUL_MAT_ID
|
| 55 |
+
uint nei0;
|
| 56 |
+
uint nei1;
|
| 57 |
+
uint nbi1;
|
| 58 |
+
uint ne11;
|
| 59 |
+
#else
|
| 60 |
+
uint base_work_group_z;
|
| 61 |
+
uint num_batches;
|
| 62 |
+
uint k_split;
|
| 63 |
+
uint ne02;
|
| 64 |
+
uint ne12;
|
| 65 |
+
uint broadcast2;
|
| 66 |
+
uint broadcast3;
|
| 67 |
+
#endif
|
| 68 |
+
} p;
|
| 69 |
+
|
| 70 |
+
layout (constant_id = 0) const uint BLOCK_SIZE = 64;
|
| 71 |
+
layout (constant_id = 1) const uint BM = 64;
|
| 72 |
+
layout (constant_id = 2) const uint BN = 64;
|
| 73 |
+
// layout (constant_id = 3) const uint BK = 32;
|
| 74 |
+
layout (constant_id = 4) const uint WM = 32;
|
| 75 |
+
layout (constant_id = 5) const uint WN = 32;
|
| 76 |
+
layout (constant_id = 6) const uint WMITER = 2;
|
| 77 |
+
layout (constant_id = 7) const uint TM = 4;
|
| 78 |
+
layout (constant_id = 8) const uint TN = 2;
|
| 79 |
+
layout (constant_id = 9) const uint TK = 1; // Only needed for coopmat
|
| 80 |
+
layout (constant_id = 10) const uint WARP = 32;
|
| 81 |
+
|
| 82 |
+
#define BK 32
|
| 83 |
+
|
| 84 |
+
#include "mul_mmq_shmem_types.glsl"
|
| 85 |
+
|
| 86 |
+
#ifdef MUL_MAT_ID
|
| 87 |
+
#define BK_STEP 1
|
| 88 |
+
#else
|
| 89 |
+
#ifndef BK_STEP
|
| 90 |
+
#define BK_STEP 4
|
| 91 |
+
#endif
|
| 92 |
+
#endif
|
| 93 |
+
|
| 94 |
+
// Shared memory cache
|
| 95 |
+
shared block_a_cache buf_a[BM * BK_STEP];
|
| 96 |
+
shared block_b_cache buf_b[BN * BK_STEP];
|
| 97 |
+
// Register cache
|
| 98 |
+
block_a_cache cache_a[WMITER * TM];
|
| 99 |
+
block_b_cache cache_b;
|
| 100 |
+
|
| 101 |
+
#define LOAD_VEC_A (4 * QUANT_R_MMQ)
|
| 102 |
+
#define LOAD_VEC_B 16
|
| 103 |
+
|
| 104 |
+
#define NUM_WARPS (BLOCK_SIZE / WARP)
|
| 105 |
+
|
| 106 |
+
#include "mul_mm_id_funcs.glsl"
|
| 107 |
+
#include "mul_mmq_funcs.glsl"
|
| 108 |
+
|
| 109 |
+
void main() {
|
| 110 |
+
const uint ic = gl_WorkGroupID.y;
|
| 111 |
+
|
| 112 |
+
#ifdef MUL_MAT_ID
|
| 113 |
+
const uint expert_idx = gl_WorkGroupID.z;
|
| 114 |
+
if (ic * BN >= data_expert_count[expert_idx]) {
|
| 115 |
+
return;
|
| 116 |
+
}
|
| 117 |
+
#endif
|
| 118 |
+
#ifdef NEEDS_INIT_IQ_SHMEM
|
| 119 |
+
init_iq_shmem(gl_WorkGroupSize);
|
| 120 |
+
#endif
|
| 121 |
+
|
| 122 |
+
#ifndef MUL_MAT_ID
|
| 123 |
+
const uint batch_idx = gl_WorkGroupID.z + p.base_work_group_z;
|
| 124 |
+
|
| 125 |
+
const uint i13 = batch_idx / p.ne12;
|
| 126 |
+
const uint i12 = batch_idx % p.ne12;
|
| 127 |
+
|
| 128 |
+
const uint i03 = i13 / p.broadcast3;
|
| 129 |
+
const uint i02 = i12 / p.broadcast2;
|
| 130 |
+
|
| 131 |
+
const uint batch_idx_a = i03 * p.ne02 + i02;
|
| 132 |
+
#endif
|
| 133 |
+
|
| 134 |
+
const uint blocks_m = (p.M + BM - 1) / BM;
|
| 135 |
+
const uint ir = gl_WorkGroupID.x % blocks_m;
|
| 136 |
+
const uint ik = gl_WorkGroupID.x / blocks_m;
|
| 137 |
+
|
| 138 |
+
const uint WNITER = (WM * WN) / (WARP * TM * TN * WMITER);
|
| 139 |
+
const uint WSUBM = WM / WMITER;
|
| 140 |
+
const uint WSUBN = WN / WNITER;
|
| 141 |
+
const uint warp_i = gl_LocalInvocationID.x / WARP;
|
| 142 |
+
|
| 143 |
+
const uint tiw = gl_LocalInvocationID.x % WARP;
|
| 144 |
+
|
| 145 |
+
const uint tiwr = tiw % (WSUBM / TM);
|
| 146 |
+
const uint tiwc = tiw / (WSUBM / TM);
|
| 147 |
+
|
| 148 |
+
const uint warp_r = warp_i % (BM / WM);
|
| 149 |
+
const uint warp_c = warp_i / (BM / WM);
|
| 150 |
+
|
| 151 |
+
const uint loadr_a = gl_LocalInvocationID.x % (BK / LOAD_VEC_A);
|
| 152 |
+
const uint loadc_a = gl_LocalInvocationID.x / (BK / LOAD_VEC_A);
|
| 153 |
+
const uint loadr_b = gl_LocalInvocationID.x % (BK / LOAD_VEC_B);
|
| 154 |
+
const uint loadc_b = gl_LocalInvocationID.x / (BK / LOAD_VEC_B);
|
| 155 |
+
|
| 156 |
+
const uint loadstride_a = BLOCK_SIZE * LOAD_VEC_A / BK;
|
| 157 |
+
const uint loadstride_b = BLOCK_SIZE * LOAD_VEC_B / BK;
|
| 158 |
+
|
| 159 |
+
#ifdef MUL_MAT_ID
|
| 160 |
+
#ifdef MUL_MAT_ID_USE_SUBGROUPS
|
| 161 |
+
if (bitCount(p.nei0) == 1) {
|
| 162 |
+
load_row_ids(expert_idx, true, ic);
|
| 163 |
+
} else {
|
| 164 |
+
load_row_ids(expert_idx, false, ic);
|
| 165 |
+
}
|
| 166 |
+
#else
|
| 167 |
+
_ne1 = 0;
|
| 168 |
+
for (uint ii1 = 0; ii1 < p.nei1 && _ne1 < (ic + 1) * BN; ii1++) {
|
| 169 |
+
for (uint ii0 = 0; ii0 < p.nei0 && _ne1 < (ic + 1) * BN; ii0++) {
|
| 170 |
+
if (data_ids[ii1*p.nbi1 + ii0] == expert_idx) {
|
| 171 |
+
if (_ne1 >= ic * BN) {
|
| 172 |
+
row_ids[_ne1 - ic * BN] = u16vec2(ii0, ii1);
|
| 173 |
+
}
|
| 174 |
+
_ne1++;
|
| 175 |
+
}
|
| 176 |
+
}
|
| 177 |
+
}
|
| 178 |
+
|
| 179 |
+
barrier();
|
| 180 |
+
#endif
|
| 181 |
+
|
| 182 |
+
// Workgroup has no work
|
| 183 |
+
if (ic * BN >= _ne1) return;
|
| 184 |
+
#endif
|
| 185 |
+
|
| 186 |
+
#ifdef MUL_MAT_ID
|
| 187 |
+
const uint start_k = 0;
|
| 188 |
+
const uint end_k = p.K;
|
| 189 |
+
#else
|
| 190 |
+
const uint start_k = ik * p.k_split;
|
| 191 |
+
const uint end_k = min(p.K, (ik + 1) * p.k_split);
|
| 192 |
+
#endif
|
| 193 |
+
|
| 194 |
+
uint pos_a_ib =
|
| 195 |
+
#ifdef MUL_MAT_ID
|
| 196 |
+
expert_idx * (p.batch_stride_a / BK) +
|
| 197 |
+
#else
|
| 198 |
+
batch_idx_a * (p.batch_stride_a / BK) +
|
| 199 |
+
#endif
|
| 200 |
+
(ir * BM * p.stride_a + start_k) / BK;
|
| 201 |
+
#ifdef MUL_MAT_ID
|
| 202 |
+
uint pos_b_ib = 0;
|
| 203 |
+
#else
|
| 204 |
+
uint pos_b_ib = (batch_idx * p.batch_stride_b + ic * BN * p.stride_b + start_k) / BK;
|
| 205 |
+
#endif
|
| 206 |
+
|
| 207 |
+
ACC_TYPE sums[WMITER * TM * WNITER * TN];
|
| 208 |
+
|
| 209 |
+
[[unroll]] for (uint i = 0; i < WMITER*TM*WNITER*TN; i++) {
|
| 210 |
+
sums[i] = ACC_TYPE(0.0f);
|
| 211 |
+
}
|
| 212 |
+
|
| 213 |
+
for (uint block = start_k; block < end_k; block += BK * BK_STEP) {
|
| 214 |
+
[[unroll]] for (uint l = 0; loadc_a + l < BM; l += loadstride_a) {
|
| 215 |
+
const uint buf_ib = loadc_a + l;
|
| 216 |
+
const uint ib = pos_a_ib + buf_ib * p.stride_a / BK;
|
| 217 |
+
const uint iqs = loadr_a;
|
| 218 |
+
|
| 219 |
+
[[unroll]] for (uint k_step = 0; k_step < BK_STEP; k_step++) {
|
| 220 |
+
if (block + k_step * BK < end_k) {
|
| 221 |
+
block_a_to_shmem(k_step * BM + buf_ib, ib + k_step, iqs);
|
| 222 |
+
}
|
| 223 |
+
}
|
| 224 |
+
}
|
| 225 |
+
[[unroll]] for (uint l = 0; loadc_b + l < BN; l += loadstride_b) {
|
| 226 |
+
const uint buf_ib = loadc_b + l;
|
| 227 |
+
|
| 228 |
+
#ifdef MUL_MAT_ID
|
| 229 |
+
const u16vec2 row_idx = row_ids[buf_ib];
|
| 230 |
+
const uint ib = pos_b_ib + row_idx.y * p.batch_stride_b / BK + (row_idx.x % p.ne11) * p.stride_b / BK;
|
| 231 |
+
#else
|
| 232 |
+
const uint ib = pos_b_ib + buf_ib * p.stride_b / BK;
|
| 233 |
+
#endif
|
| 234 |
+
const uint iqs = loadr_b;
|
| 235 |
+
|
| 236 |
+
[[unroll]] for (uint k_step = 0; k_step < BK_STEP; k_step++) {
|
| 237 |
+
block_b_to_shmem(k_step * BN + buf_ib, ib + k_step, iqs, block + k_step * BK < end_k);
|
| 238 |
+
}
|
| 239 |
+
}
|
| 240 |
+
|
| 241 |
+
barrier();
|
| 242 |
+
|
| 243 |
+
pos_a_ib += BK_STEP;
|
| 244 |
+
pos_b_ib += BK_STEP;
|
| 245 |
+
|
| 246 |
+
for (uint k_step = 0; k_step < BK_STEP; k_step++) {
|
| 247 |
+
// Load from shared into cache
|
| 248 |
+
[[unroll]] for (uint wsir = 0; wsir < WMITER; wsir++) {
|
| 249 |
+
[[unroll]] for (uint cr = 0; cr < TM; cr++) {
|
| 250 |
+
const uint reg_ib = wsir * TM + cr;
|
| 251 |
+
const uint buf_ib = warp_r * WM + wsir * WSUBM + tiwr * TM + cr;
|
| 252 |
+
|
| 253 |
+
block_a_to_registers(reg_ib, k_step * BM + buf_ib);
|
| 254 |
+
}
|
| 255 |
+
}
|
| 256 |
+
|
| 257 |
+
[[unroll]] for (uint wsic = 0; wsic < WNITER; wsic++) {
|
| 258 |
+
[[unroll]] for (uint cc = 0; cc < TN; cc++) {
|
| 259 |
+
const uint ib = k_step * BN + warp_c * WN + wsic * WSUBN + tiwc * TN + cc;
|
| 260 |
+
block_b_to_registers(ib);
|
| 261 |
+
|
| 262 |
+
[[unroll]] for (uint wsir = 0; wsir < WMITER; wsir++) {
|
| 263 |
+
[[unroll]] for (uint cr = 0; cr < TM; cr++) {
|
| 264 |
+
const uint cache_a_idx = wsir * TM + cr;
|
| 265 |
+
const uint sums_idx = (wsic * TN + cc) * (WMITER * TM) + wsir * TM + cr;
|
| 266 |
+
|
| 267 |
+
sums[sums_idx] += mmq_dot_product(cache_a_idx);
|
| 268 |
+
}
|
| 269 |
+
}
|
| 270 |
+
}
|
| 271 |
+
}
|
| 272 |
+
}
|
| 273 |
+
|
| 274 |
+
barrier();
|
| 275 |
+
}
|
| 276 |
+
|
| 277 |
+
const uint dr = ir * BM + warp_r * WM;
|
| 278 |
+
const uint dc = ic * BN + warp_c * WN;
|
| 279 |
+
|
| 280 |
+
#ifndef MUL_MAT_ID
|
| 281 |
+
const uint offsets = batch_idx * p.batch_stride_d + ik * p.batch_stride_d * p.num_batches;
|
| 282 |
+
#endif
|
| 283 |
+
|
| 284 |
+
[[unroll]] for (uint wsic = 0; wsic < WNITER; wsic++) {
|
| 285 |
+
[[unroll]] for (uint wsir = 0; wsir < WMITER; wsir++) {
|
| 286 |
+
|
| 287 |
+
const uint dr_warp = dr + wsir * WSUBM + tiwr * TM;
|
| 288 |
+
const uint dc_warp = dc + wsic * WSUBN + tiwc * TN;
|
| 289 |
+
[[unroll]] for (uint cc = 0; cc < TN; cc++) {
|
| 290 |
+
#ifdef MUL_MAT_ID
|
| 291 |
+
const uint row_i = dc_warp + cc;
|
| 292 |
+
if (row_i >= _ne1) break;
|
| 293 |
+
|
| 294 |
+
const u16vec2 row_idx = row_ids[row_i - ic * BN];
|
| 295 |
+
#endif // MUL_MAT_ID
|
| 296 |
+
[[unroll]] for (uint cr = 0; cr < TM; cr++) {
|
| 297 |
+
const uint sums_idx = (wsic * TN + cc) * WMITER * TM + wsir * TM + cr;
|
| 298 |
+
#ifdef MUL_MAT_ID
|
| 299 |
+
if (dr_warp + cr < p.M) {
|
| 300 |
+
data_d[row_idx.y * p.batch_stride_d + row_idx.x * p.stride_d + dr_warp + cr] = D_TYPE(sums[sums_idx].x);
|
| 301 |
+
}
|
| 302 |
+
#else
|
| 303 |
+
if (dr_warp + cr < p.M && dc_warp + cc < p.N) {
|
| 304 |
+
data_d[offsets + (dc_warp + cc) * p.stride_d + dr_warp + cr] = D_TYPE(sums[sums_idx].x);
|
| 305 |
+
}
|
| 306 |
+
#endif // MUL_MAT_ID
|
| 307 |
+
}
|
| 308 |
+
}
|
| 309 |
+
}
|
| 310 |
+
}
|
| 311 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_funcs.glsl
ADDED
|
@@ -0,0 +1,454 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#extension GL_EXT_shader_explicit_arithmetic_types_int32 : require
|
| 2 |
+
#extension GL_EXT_shader_explicit_arithmetic_types_int16 : require
|
| 3 |
+
#extension GL_EXT_shader_explicit_arithmetic_types_int8 : require
|
| 4 |
+
|
| 5 |
+
#include "types.glsl"
|
| 6 |
+
|
| 7 |
+
// Each iqs value maps to a 32-bit integer
|
| 8 |
+
|
| 9 |
+
#if defined(DATA_A_Q4_0) || defined(DATA_A_Q4_1)
|
| 10 |
+
// 2-byte loads for Q4_0 blocks (18 bytes)
|
| 11 |
+
// 4-byte loads for Q4_1 blocks (20 bytes)
|
| 12 |
+
void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) {
|
| 13 |
+
#ifdef DATA_A_Q4_0
|
| 14 |
+
buf_a[buf_ib].qs[iqs] = pack32(u16vec2(data_a_packed16[ib].qs[iqs * 2],
|
| 15 |
+
data_a_packed16[ib].qs[iqs * 2 + 1]));
|
| 16 |
+
|
| 17 |
+
if (iqs == 0) {
|
| 18 |
+
buf_a[buf_ib].dm = FLOAT_TYPE(data_a_packed16[ib].d);
|
| 19 |
+
}
|
| 20 |
+
#else // DATA_A_Q4_1
|
| 21 |
+
buf_a[buf_ib].qs[iqs] = data_a_packed32[ib].qs[iqs];
|
| 22 |
+
|
| 23 |
+
if (iqs == 0) {
|
| 24 |
+
buf_a[buf_ib].dm = FLOAT_TYPEV2(data_a_packed32[ib].dm);
|
| 25 |
+
}
|
| 26 |
+
#endif
|
| 27 |
+
}
|
| 28 |
+
|
| 29 |
+
void block_a_to_registers(const uint reg_ib, const uint buf_ib) {
|
| 30 |
+
cache_a[reg_ib].dm = buf_a[buf_ib].dm;
|
| 31 |
+
|
| 32 |
+
[[unroll]] for (uint iqs = 0; iqs < 4; iqs++) {
|
| 33 |
+
cache_a[reg_ib].qs[iqs] = buf_a[buf_ib].qs[iqs];
|
| 34 |
+
}
|
| 35 |
+
}
|
| 36 |
+
|
| 37 |
+
ACC_TYPE mmq_dot_product(const uint ib_a) {
|
| 38 |
+
int32_t q_sum = 0;
|
| 39 |
+
[[unroll]] for (uint iqs = 0; iqs < 4; iqs++) {
|
| 40 |
+
const uint32_t vui = cache_a[ib_a].qs[iqs];
|
| 41 |
+
const i32vec2 qs_a = i32vec2( vui & 0x0F0F0F0F,
|
| 42 |
+
(vui >> 4) & 0x0F0F0F0F);
|
| 43 |
+
|
| 44 |
+
const int32_t qs_b0 = cache_b.qs[iqs];
|
| 45 |
+
const int32_t qs_b1 = cache_b.qs[iqs + 4];
|
| 46 |
+
|
| 47 |
+
q_sum += dotPacked4x8EXT(qs_a.x, qs_b0);
|
| 48 |
+
q_sum += dotPacked4x8EXT(qs_a.y, qs_b1);
|
| 49 |
+
}
|
| 50 |
+
|
| 51 |
+
#ifdef DATA_A_Q4_0
|
| 52 |
+
return ACC_TYPE(float(cache_a[ib_a].dm) * (float(q_sum) * float(cache_b.ds.x) - 8.0 * float(cache_b.ds.y)));
|
| 53 |
+
#else // DATA_A_Q4_1
|
| 54 |
+
return ACC_TYPE(float(q_sum) * float(cache_a[ib_a].dm.x) * float(cache_b.ds.x) + float(cache_a[ib_a].dm.y) * float(cache_b.ds.y));
|
| 55 |
+
#endif
|
| 56 |
+
}
|
| 57 |
+
#endif
|
| 58 |
+
|
| 59 |
+
#if defined(DATA_A_Q5_0) || defined(DATA_A_Q5_1)
|
| 60 |
+
// 2-byte loads for Q5_0 blocks (22 bytes)
|
| 61 |
+
// 4-byte loads for Q5_1 blocks (24 bytes)
|
| 62 |
+
void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) {
|
| 63 |
+
#ifdef DATA_A_Q5_0
|
| 64 |
+
buf_a[buf_ib].qs[iqs] = pack32(u16vec2(data_a_packed16[ib].qs[iqs * 2],
|
| 65 |
+
data_a_packed16[ib].qs[iqs * 2 + 1]));
|
| 66 |
+
|
| 67 |
+
if (iqs == 0) {
|
| 68 |
+
buf_a[buf_ib].dm = FLOAT_TYPE(data_a_packed16[ib].d);
|
| 69 |
+
buf_a[buf_ib].qh = pack32(u16vec2(data_a_packed16[ib].qh[0], data_a_packed16[ib].qh[1]));
|
| 70 |
+
}
|
| 71 |
+
#else // DATA_A_Q5_1
|
| 72 |
+
buf_a[buf_ib].qs[iqs] = data_a_packed32[ib].qs[iqs];
|
| 73 |
+
|
| 74 |
+
if (iqs == 0) {
|
| 75 |
+
buf_a[buf_ib].dm = FLOAT_TYPEV2(data_a_packed32[ib].dm);
|
| 76 |
+
buf_a[buf_ib].qh = data_a_packed32[ib].qh;
|
| 77 |
+
}
|
| 78 |
+
#endif
|
| 79 |
+
}
|
| 80 |
+
|
| 81 |
+
void block_a_to_registers(const uint reg_ib, const uint buf_ib) {
|
| 82 |
+
cache_a[reg_ib].dm = buf_a[buf_ib].dm;
|
| 83 |
+
cache_a[reg_ib].qh = buf_a[buf_ib].qh;
|
| 84 |
+
|
| 85 |
+
[[unroll]] for (uint iqs = 0; iqs < 4; iqs++) {
|
| 86 |
+
cache_a[reg_ib].qs[iqs] = buf_a[buf_ib].qs[iqs];
|
| 87 |
+
}
|
| 88 |
+
}
|
| 89 |
+
|
| 90 |
+
ACC_TYPE mmq_dot_product(const uint ib_a) {
|
| 91 |
+
int32_t q_sum = 0;
|
| 92 |
+
[[unroll]] for (uint iqs = 0; iqs < 4; iqs++) {
|
| 93 |
+
const uint32_t vui = cache_a[ib_a].qs[iqs];
|
| 94 |
+
const int32_t qh = int32_t(cache_a[ib_a].qh >> (4 * iqs));
|
| 95 |
+
const int32_t qs_a0 = int32_t(vui & 0x0F0F0F0F)
|
| 96 |
+
| ((qh & 0xF) * 0x02040810) & 0x10101010; // (0,1,2,3) -> (4,12,20,28)
|
| 97 |
+
const int32_t qs_a1 = int32_t((vui >> 4) & 0x0F0F0F0F)
|
| 98 |
+
| (((qh >> 16) & 0xF) * 0x02040810) & 0x10101010; // (16,17,18,19) -> (4,12,20,28)
|
| 99 |
+
|
| 100 |
+
const int32_t qs_b0 = cache_b.qs[iqs];
|
| 101 |
+
const int32_t qs_b1 = cache_b.qs[iqs + 4];
|
| 102 |
+
|
| 103 |
+
q_sum += dotPacked4x8EXT(qs_a0, qs_b0);
|
| 104 |
+
q_sum += dotPacked4x8EXT(qs_a1, qs_b1);
|
| 105 |
+
}
|
| 106 |
+
|
| 107 |
+
#ifdef DATA_A_Q5_0
|
| 108 |
+
return ACC_TYPE(float(cache_a[ib_a].dm) * (float(q_sum) * float(cache_b.ds.x) - 16.0 * float(cache_b.ds.y)));
|
| 109 |
+
#else // DATA_A_Q5_1
|
| 110 |
+
return ACC_TYPE(float(q_sum) * float(cache_a[ib_a].dm.x) * float(cache_b.ds.x) + float(cache_a[ib_a].dm.y) * float(cache_b.ds.y));
|
| 111 |
+
#endif
|
| 112 |
+
}
|
| 113 |
+
#endif
|
| 114 |
+
|
| 115 |
+
#if defined(DATA_A_Q8_0)
|
| 116 |
+
// 2-byte loads for Q8_0 blocks (34 bytes)
|
| 117 |
+
void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) {
|
| 118 |
+
buf_a[buf_ib].qs[iqs] = pack32(i16vec2(data_a_packed16[ib].qs[iqs * 2],
|
| 119 |
+
data_a_packed16[ib].qs[iqs * 2 + 1]));
|
| 120 |
+
|
| 121 |
+
if (iqs == 0) {
|
| 122 |
+
buf_a[buf_ib].dm = FLOAT_TYPE(data_a_packed16[ib].d);
|
| 123 |
+
}
|
| 124 |
+
}
|
| 125 |
+
|
| 126 |
+
void block_a_to_registers(const uint reg_ib, const uint buf_ib) {
|
| 127 |
+
cache_a[reg_ib].dm = buf_a[buf_ib].dm;
|
| 128 |
+
|
| 129 |
+
[[unroll]] for (uint iqs = 0; iqs < 8; iqs++) {
|
| 130 |
+
cache_a[reg_ib].qs[iqs] = buf_a[buf_ib].qs[iqs];
|
| 131 |
+
}
|
| 132 |
+
}
|
| 133 |
+
|
| 134 |
+
ACC_TYPE mmq_dot_product(const uint ib_a) {
|
| 135 |
+
int32_t q_sum = 0;
|
| 136 |
+
[[unroll]] for (uint iqs = 0; iqs < 8; iqs++) {
|
| 137 |
+
const int32_t qs_a = cache_a[ib_a].qs[iqs];
|
| 138 |
+
const int32_t qs_b = cache_b.qs[iqs];
|
| 139 |
+
|
| 140 |
+
q_sum += dotPacked4x8EXT(qs_a, qs_b);
|
| 141 |
+
}
|
| 142 |
+
|
| 143 |
+
return ACC_TYPE(float(q_sum) * float(cache_a[ib_a].dm) * float(cache_b.ds.x));
|
| 144 |
+
}
|
| 145 |
+
#endif
|
| 146 |
+
|
| 147 |
+
#if defined(DATA_A_MXFP4)
|
| 148 |
+
// 1-byte loads for mxfp4 blocks (17 bytes)
|
| 149 |
+
void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) {
|
| 150 |
+
const uint32_t qs = pack32(u8vec4(data_a[ib].qs[iqs * 4 ],
|
| 151 |
+
data_a[ib].qs[iqs * 4 + 1],
|
| 152 |
+
data_a[ib].qs[iqs * 4 + 2],
|
| 153 |
+
data_a[ib].qs[iqs * 4 + 3]));
|
| 154 |
+
|
| 155 |
+
const u8vec4 i_a0 = unpack8( qs & 0x0F0F0F0F);
|
| 156 |
+
const u8vec4 i_a1 = unpack8((qs >> 4) & 0x0F0F0F0F);
|
| 157 |
+
|
| 158 |
+
buf_a[buf_ib].qs[iqs ] = pack32(i8vec4(kvalues_mxfp4[i_a0.x], kvalues_mxfp4[i_a0.y], kvalues_mxfp4[i_a0.z], kvalues_mxfp4[i_a0.w]));
|
| 159 |
+
buf_a[buf_ib].qs[iqs + 4] = pack32(i8vec4(kvalues_mxfp4[i_a1.x], kvalues_mxfp4[i_a1.y], kvalues_mxfp4[i_a1.z], kvalues_mxfp4[i_a1.w]));
|
| 160 |
+
|
| 161 |
+
if (iqs == 0) {
|
| 162 |
+
buf_a[buf_ib].d = FLOAT_TYPE(e8m0_to_fp32(data_a[ib].e) * 0.5);
|
| 163 |
+
}
|
| 164 |
+
}
|
| 165 |
+
|
| 166 |
+
void block_a_to_registers(const uint reg_ib, const uint buf_ib) {
|
| 167 |
+
cache_a[reg_ib].d = buf_a[buf_ib].d;
|
| 168 |
+
|
| 169 |
+
[[unroll]] for (uint iqs = 0; iqs < 8; iqs++) {
|
| 170 |
+
cache_a[reg_ib].qs[iqs] = buf_a[buf_ib].qs[iqs];
|
| 171 |
+
}
|
| 172 |
+
}
|
| 173 |
+
|
| 174 |
+
ACC_TYPE mmq_dot_product(const uint ib_a) {
|
| 175 |
+
int32_t q_sum = 0;
|
| 176 |
+
[[unroll]] for (uint iqs = 0; iqs < 8; iqs++) {
|
| 177 |
+
const int32_t qs_a = cache_a[ib_a].qs[iqs];
|
| 178 |
+
|
| 179 |
+
q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
|
| 180 |
+
}
|
| 181 |
+
|
| 182 |
+
return ACC_TYPE(float(cache_a[ib_a].d) * float(cache_b.ds.x) * float(q_sum));
|
| 183 |
+
}
|
| 184 |
+
#endif
|
| 185 |
+
|
| 186 |
+
// For k-quants, ib and iqs still assume 32-wide blocks, but k-quants are 256-wide
|
| 187 |
+
// iqs still refers to a 32-bit integer, meaning 0..7 for 32-wide quants
|
| 188 |
+
#if defined(DATA_A_Q2_K)
|
| 189 |
+
// 4-byte loads for Q2_K blocks (84 bytes)
|
| 190 |
+
void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) {
|
| 191 |
+
const uint ib_k = ib / 8;
|
| 192 |
+
const uint iqs_k = (ib % 8) * 8 + iqs * QUANT_R_MMQ;
|
| 193 |
+
|
| 194 |
+
const uint qs_idx = (iqs_k / 32) * 8 + (iqs_k % 8);
|
| 195 |
+
const uint qs_shift = ((iqs_k % 32) / 8) * 2;
|
| 196 |
+
|
| 197 |
+
// Repack 4x4 quants into one int
|
| 198 |
+
const uint32_t vals0 = (data_a_packed32[ib_k].qs[qs_idx ] >> qs_shift) & 0x03030303;
|
| 199 |
+
const uint32_t vals1 = (data_a_packed32[ib_k].qs[qs_idx + 1] >> qs_shift) & 0x03030303;
|
| 200 |
+
const uint32_t vals2 = (data_a_packed32[ib_k].qs[qs_idx + 2] >> qs_shift) & 0x03030303;
|
| 201 |
+
const uint32_t vals3 = (data_a_packed32[ib_k].qs[qs_idx + 3] >> qs_shift) & 0x03030303;
|
| 202 |
+
|
| 203 |
+
buf_a[buf_ib].qs[iqs] = vals0 | (vals1 << 2) | (vals2 << 4) | (vals3 << 6);
|
| 204 |
+
|
| 205 |
+
if (iqs == 0) {
|
| 206 |
+
buf_a[buf_ib].dm = FLOAT_TYPEV2(data_a_packed32[ib_k].dm);
|
| 207 |
+
buf_a[buf_ib].scales = unpack8(uint32_t(data_a_packed16[ib_k].scales[iqs_k / 8])).xy; // vec4 used due to #12147
|
| 208 |
+
}
|
| 209 |
+
}
|
| 210 |
+
|
| 211 |
+
void block_a_to_registers(const uint reg_ib, const uint buf_ib) {
|
| 212 |
+
cache_a[reg_ib].dm = buf_a[buf_ib].dm;
|
| 213 |
+
cache_a[reg_ib].scales = buf_a[buf_ib].scales;
|
| 214 |
+
|
| 215 |
+
[[unroll]] for (uint iqs = 0; iqs < 2; iqs++) {
|
| 216 |
+
cache_a[reg_ib].qs[iqs] = buf_a[buf_ib].qs[iqs];
|
| 217 |
+
}
|
| 218 |
+
}
|
| 219 |
+
|
| 220 |
+
ACC_TYPE mmq_dot_product(const uint ib_a) {
|
| 221 |
+
int32_t sum_d = 0;
|
| 222 |
+
int32_t sum_m = 0;
|
| 223 |
+
|
| 224 |
+
[[unroll]] for (uint iqs = 0; iqs < 8; iqs++) {
|
| 225 |
+
const uint8_t scale = cache_a[ib_a].scales[iqs / 4];
|
| 226 |
+
const int32_t scale_m = int32_t(scale >> 4) * 0x01010101; // Duplicate 8-bit value across 32-bits.
|
| 227 |
+
const int32_t qs_a = int32_t((cache_a[ib_a].qs[iqs / 4] >> ((iqs % 4) * 2)) & 0x03030303);
|
| 228 |
+
|
| 229 |
+
sum_d += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]) * (scale & 0xF);
|
| 230 |
+
sum_m += dotPacked4x8EXT(scale_m, cache_b.qs[iqs]);
|
| 231 |
+
}
|
| 232 |
+
|
| 233 |
+
return ACC_TYPE(float(cache_b.ds.x) * (float(cache_a[ib_a].dm.x) * float(sum_d) - float(cache_a[ib_a].dm.y) * float(sum_m)));
|
| 234 |
+
}
|
| 235 |
+
#endif
|
| 236 |
+
|
| 237 |
+
#if defined(DATA_A_Q3_K)
|
| 238 |
+
// 2-byte loads for Q3_K blocks (110 bytes)
|
| 239 |
+
void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) {
|
| 240 |
+
const uint ib_k = ib / 8;
|
| 241 |
+
const uint hm_idx = iqs * QUANT_R_MMQ;
|
| 242 |
+
const uint iqs_k = (ib % 8) * 8 + hm_idx;
|
| 243 |
+
|
| 244 |
+
const uint qs_idx = (iqs_k / 32) * 8 + (iqs_k % 8);
|
| 245 |
+
const uint qs_shift = ((iqs_k % 32) / 8) * 2;
|
| 246 |
+
const uint hm_shift = iqs_k / 8;
|
| 247 |
+
|
| 248 |
+
// Repack 2x4 quants into one int
|
| 249 |
+
// Add the 3rd bit instead of subtracting it to allow packing the quants
|
| 250 |
+
// vec4 for unpack8 used due to #12147
|
| 251 |
+
const i8vec2 vals00 = unpack8(int32_t(int16_t((data_a_packed16[ib_k].qs[qs_idx * 2 ] >> qs_shift) & uint16_t(0x0303)))).xy |
|
| 252 |
+
unpack8(int32_t(int16_t(((data_a_packed16[ib_k].hmask[hm_idx * 2 ] >> hm_shift) & uint16_t(0x0101))) << 2)).xy;
|
| 253 |
+
const i8vec2 vals01 = unpack8(int32_t(int16_t((data_a_packed16[ib_k].qs[qs_idx * 2 + 1 ] >> qs_shift) & uint16_t(0x0303)))).xy |
|
| 254 |
+
unpack8(int32_t(int16_t(((data_a_packed16[ib_k].hmask[hm_idx * 2 + 1] >> hm_shift) & uint16_t(0x0101))) << 2)).xy;
|
| 255 |
+
const i8vec2 vals10 = unpack8(int32_t(int16_t((data_a_packed16[ib_k].qs[qs_idx * 2 + 2 ] >> qs_shift) & uint16_t(0x0303)))).xy |
|
| 256 |
+
unpack8(int32_t(int16_t(((data_a_packed16[ib_k].hmask[hm_idx * 2 + 2] >> hm_shift) & uint16_t(0x0101))) << 2)).xy;
|
| 257 |
+
const i8vec2 vals11 = unpack8(int32_t(int16_t((data_a_packed16[ib_k].qs[qs_idx * 2 + 3 ] >> qs_shift) & uint16_t(0x0303)))).xy |
|
| 258 |
+
unpack8(int32_t(int16_t(((data_a_packed16[ib_k].hmask[hm_idx * 2 + 3] >> hm_shift) & uint16_t(0x0101))) << 2)).xy;
|
| 259 |
+
buf_a[buf_ib].qs[iqs] = pack32(u8vec4(vals00.x, vals00.y, vals01.x, vals01.y)) |
|
| 260 |
+
(pack32(u8vec4(vals10.x, vals10.y, vals11.x, vals11.y)) << 4);
|
| 261 |
+
|
| 262 |
+
if (iqs == 0) {
|
| 263 |
+
const uint is = iqs_k / 4;
|
| 264 |
+
const i8vec2 scales = i8vec2(unpack8(uint32_t(((data_a_packed16[ib_k].scales[(is % 8 ) / 2] >> (4 * (is / 8))) & 0x0F0F) |
|
| 265 |
+
(((data_a_packed16[ib_k].scales[(8 + (is % 4)) / 2] >> (2 * (is / 4))) & 0x0303) << 4))).xy); // vec4 used due to #12147
|
| 266 |
+
|
| 267 |
+
buf_a[buf_ib].d_scales = FLOAT_TYPEV2(float(data_a_packed16[ib_k].d) * vec2(scales - 32));
|
| 268 |
+
}
|
| 269 |
+
}
|
| 270 |
+
|
| 271 |
+
void block_a_to_registers(const uint reg_ib, const uint buf_ib) {
|
| 272 |
+
cache_a[reg_ib].d_scales = buf_a[buf_ib].d_scales;
|
| 273 |
+
|
| 274 |
+
[[unroll]] for (uint iqs = 0; iqs < 4; iqs++) {
|
| 275 |
+
cache_a[reg_ib].qs[iqs] = buf_a[buf_ib].qs[iqs];
|
| 276 |
+
}
|
| 277 |
+
}
|
| 278 |
+
|
| 279 |
+
ACC_TYPE mmq_dot_product(const uint ib_a) {
|
| 280 |
+
float result = 0.0;
|
| 281 |
+
int32_t q_sum = 0;
|
| 282 |
+
|
| 283 |
+
[[unroll]] for (uint iqs = 0; iqs < 4; iqs++) {
|
| 284 |
+
// Subtract 4 from the quants to correct the 3rd bit offset
|
| 285 |
+
const int32_t qs_a = pack32(unpack8(int32_t((cache_a[ib_a].qs[iqs / 2] >> ((iqs % 2) * 4)) & 0x0F0F0F0F)) - int8_t(4));
|
| 286 |
+
|
| 287 |
+
q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
|
| 288 |
+
}
|
| 289 |
+
result += float(cache_a[ib_a].d_scales[0]) * float(q_sum);
|
| 290 |
+
q_sum = 0;
|
| 291 |
+
|
| 292 |
+
[[unroll]] for (uint iqs = 4; iqs < 8; iqs++) {
|
| 293 |
+
const int32_t qs_a = pack32(unpack8(int32_t((cache_a[ib_a].qs[iqs / 2] >> ((iqs % 2) * 4)) & 0x0F0F0F0F)) - int8_t(4));
|
| 294 |
+
|
| 295 |
+
q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
|
| 296 |
+
}
|
| 297 |
+
result += float(cache_a[ib_a].d_scales[1]) * float(q_sum);
|
| 298 |
+
|
| 299 |
+
return ACC_TYPE(float(cache_b.ds.x) * result);
|
| 300 |
+
}
|
| 301 |
+
#endif
|
| 302 |
+
|
| 303 |
+
#if defined(DATA_A_Q4_K) || defined(DATA_A_Q5_K)
|
| 304 |
+
// 4-byte loads for Q4_K blocks (144 bytes) and Q5_K blocks (176 bytes)
|
| 305 |
+
void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) {
|
| 306 |
+
const uint ib_k = ib / 8;
|
| 307 |
+
const uint iqs_k = (ib % 8) * 8 + iqs * QUANT_R_MMQ;
|
| 308 |
+
|
| 309 |
+
const uint qs_idx = (iqs_k / 16) * 8 + (iqs_k % 8);
|
| 310 |
+
const uint qs_shift = ((iqs_k % 16) / 8) * 4;
|
| 311 |
+
|
| 312 |
+
// Repack 2x4 quants into one int
|
| 313 |
+
#if defined(DATA_A_Q4_K)
|
| 314 |
+
const uint32_t vals0 = (data_a_packed32[ib_k].qs[qs_idx ] >> qs_shift) & 0x0F0F0F0F;
|
| 315 |
+
const uint32_t vals1 = (data_a_packed32[ib_k].qs[qs_idx + 1] >> qs_shift) & 0x0F0F0F0F;
|
| 316 |
+
|
| 317 |
+
buf_a[buf_ib].qs[iqs] = vals0 | (vals1 << 4);
|
| 318 |
+
#else // defined(DATA_A_Q5_K)
|
| 319 |
+
const uint qh_idx = iqs * QUANT_R_MMQ;
|
| 320 |
+
const uint qh_shift = iqs_k / 8;
|
| 321 |
+
|
| 322 |
+
buf_a[buf_ib].qs[iqs] = int32_t(((data_a_packed32[ib_k].qs[qs_idx] >> qs_shift) & 0x0F0F0F0F) |
|
| 323 |
+
(((data_a_packed32[ib_k].qh[qh_idx] >> qh_shift) & 0x01010101) << 4));
|
| 324 |
+
#endif
|
| 325 |
+
|
| 326 |
+
if (iqs == 0) {
|
| 327 |
+
// Scale index
|
| 328 |
+
const uint is = iqs_k / 8;
|
| 329 |
+
u8vec2 scale_dm;
|
| 330 |
+
if (is < 4) {
|
| 331 |
+
scale_dm = u8vec2(data_a[ib_k].scales[is] & 0x3F, data_a[ib_k].scales[is + 4] & 0x3F);
|
| 332 |
+
} else {
|
| 333 |
+
scale_dm = u8vec2((data_a[ib_k].scales[is+4] & 0xF) | ((data_a[ib_k].scales[is-4] & 0xC0) >> 2),
|
| 334 |
+
(data_a[ib_k].scales[is+4] >> 4) | ((data_a[ib_k].scales[is ] & 0xC0) >> 2));
|
| 335 |
+
}
|
| 336 |
+
|
| 337 |
+
buf_a[buf_ib].dm = FLOAT_TYPEV2(vec2(data_a_packed32[ib_k].dm) * vec2(scale_dm));
|
| 338 |
+
}
|
| 339 |
+
}
|
| 340 |
+
|
| 341 |
+
void block_a_to_registers(const uint reg_ib, const uint buf_ib) {
|
| 342 |
+
cache_a[reg_ib].dm = buf_a[buf_ib].dm;
|
| 343 |
+
|
| 344 |
+
[[unroll]] for (uint iqs = 0; iqs < 8 / QUANT_R_MMQ; iqs++) {
|
| 345 |
+
cache_a[reg_ib].qs[iqs] = buf_a[buf_ib].qs[iqs];
|
| 346 |
+
}
|
| 347 |
+
}
|
| 348 |
+
|
| 349 |
+
ACC_TYPE mmq_dot_product(const uint ib_a) {
|
| 350 |
+
int32_t q_sum = 0;
|
| 351 |
+
|
| 352 |
+
[[unroll]] for (uint iqs = 0; iqs < 8; iqs++) {
|
| 353 |
+
#if defined(DATA_A_Q4_K)
|
| 354 |
+
const int32_t qs_a = int32_t((cache_a[ib_a].qs[iqs / 2] >> ((iqs % 2) * 4)) & 0x0F0F0F0F);
|
| 355 |
+
#else // defined(DATA_A_Q5_K)
|
| 356 |
+
const int32_t qs_a = cache_a[ib_a].qs[iqs];
|
| 357 |
+
#endif
|
| 358 |
+
|
| 359 |
+
q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
|
| 360 |
+
}
|
| 361 |
+
|
| 362 |
+
return ACC_TYPE(float(cache_b.ds.x) * float(cache_a[ib_a].dm.x) * float(q_sum) - float(cache_a[ib_a].dm.y) * float(cache_b.ds.y));
|
| 363 |
+
}
|
| 364 |
+
#endif
|
| 365 |
+
|
| 366 |
+
#if defined(DATA_A_Q6_K)
|
| 367 |
+
// 2-byte loads for Q6_K blocks (210 bytes)
|
| 368 |
+
void block_a_to_shmem(const uint buf_ib, const uint ib, const uint iqs) {
|
| 369 |
+
const uint ib_k = ib / 8;
|
| 370 |
+
const uint iqs_k = (ib % 8) * 8 + iqs;
|
| 371 |
+
|
| 372 |
+
const uint ql_idx = (iqs_k / 32) * 16 + iqs_k % 16;
|
| 373 |
+
const uint ql_shift = ((iqs_k % 32) / 16) * 4;
|
| 374 |
+
|
| 375 |
+
const uint qh_idx = (iqs_k / 32) * 8 + iqs;
|
| 376 |
+
const uint qh_shift = ((iqs_k % 32) / 8) * 2;
|
| 377 |
+
|
| 378 |
+
const i8vec2 vals00 = (unpack8(int32_t((data_a_packed16[ib_k].ql[ql_idx * 2 ] >> ql_shift) & uint16_t(0x0F0F))).xy |
|
| 379 |
+
unpack8(int32_t(((data_a_packed16[ib_k].qh[qh_idx * 2 ] >> qh_shift) & uint16_t(0x0303)) << 4)).xy) - int8_t(32);
|
| 380 |
+
const i8vec2 vals01 = (unpack8(int32_t((data_a_packed16[ib_k].ql[ql_idx * 2 + 1] >> ql_shift) & uint16_t(0x0F0F))).xy |
|
| 381 |
+
unpack8(int32_t(((data_a_packed16[ib_k].qh[qh_idx * 2 + 1] >> qh_shift) & uint16_t(0x0303)) << 4)).xy) - int8_t(32);
|
| 382 |
+
buf_a[buf_ib].qs[iqs] = pack32(i8vec4(vals00.x, vals00.y, vals01.x, vals01.y));
|
| 383 |
+
|
| 384 |
+
if (iqs == 0) {
|
| 385 |
+
const uint is = iqs_k / 4;
|
| 386 |
+
const i8vec2 scales = unpack8(int32_t(data_a_packed16[ib_k].scales[is / 2])).xy;
|
| 387 |
+
|
| 388 |
+
buf_a[buf_ib].d_scales = FLOAT_TYPEV2(float(data_a_packed16[ib_k].d) * vec2(scales));
|
| 389 |
+
}
|
| 390 |
+
}
|
| 391 |
+
|
| 392 |
+
void block_a_to_registers(const uint reg_ib, const uint buf_ib) {
|
| 393 |
+
cache_a[reg_ib].d_scales = buf_a[buf_ib].d_scales;
|
| 394 |
+
|
| 395 |
+
[[unroll]] for (uint iqs = 0; iqs < 8; iqs++) {
|
| 396 |
+
cache_a[reg_ib].qs[iqs] = buf_a[buf_ib].qs[iqs];
|
| 397 |
+
}
|
| 398 |
+
}
|
| 399 |
+
|
| 400 |
+
ACC_TYPE mmq_dot_product(const uint ib_a) {
|
| 401 |
+
float result = 0.0;
|
| 402 |
+
int32_t q_sum = 0;
|
| 403 |
+
|
| 404 |
+
[[unroll]] for (uint iqs = 0; iqs < 4; iqs++) {
|
| 405 |
+
const int32_t qs_a = cache_a[ib_a].qs[iqs];
|
| 406 |
+
|
| 407 |
+
q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
|
| 408 |
+
}
|
| 409 |
+
result += float(cache_a[ib_a].d_scales[0]) * float(q_sum);
|
| 410 |
+
q_sum = 0;
|
| 411 |
+
|
| 412 |
+
[[unroll]] for (uint iqs = 4; iqs < 8; iqs++) {
|
| 413 |
+
const int32_t qs_a = cache_a[ib_a].qs[iqs];
|
| 414 |
+
|
| 415 |
+
q_sum += dotPacked4x8EXT(qs_a, cache_b.qs[iqs]);
|
| 416 |
+
}
|
| 417 |
+
result += float(cache_a[ib_a].d_scales[1]) * float(q_sum);
|
| 418 |
+
|
| 419 |
+
return ACC_TYPE(float(cache_b.ds.x) * result);
|
| 420 |
+
}
|
| 421 |
+
#endif
|
| 422 |
+
|
| 423 |
+
void block_b_to_shmem(const uint buf_ib, const uint ib, const uint iqs, const bool is_in_bounds) {
|
| 424 |
+
if (is_in_bounds) {
|
| 425 |
+
const uint ib_outer = ib / 4;
|
| 426 |
+
const uint ib_inner = ib % 4;
|
| 427 |
+
|
| 428 |
+
if (iqs == 0) {
|
| 429 |
+
buf_b[buf_ib].ds = FLOAT_TYPEV2(data_b[ib_outer].ds[ib_inner]);
|
| 430 |
+
}
|
| 431 |
+
|
| 432 |
+
const ivec4 values = data_b[ib_outer].qs[ib_inner * 2 + iqs];
|
| 433 |
+
buf_b[buf_ib].qs[iqs * 4 ] = values.x;
|
| 434 |
+
buf_b[buf_ib].qs[iqs * 4 + 1] = values.y;
|
| 435 |
+
buf_b[buf_ib].qs[iqs * 4 + 2] = values.z;
|
| 436 |
+
buf_b[buf_ib].qs[iqs * 4 + 3] = values.w;
|
| 437 |
+
} else {
|
| 438 |
+
if (iqs == 0) {
|
| 439 |
+
buf_b[buf_ib].ds = FLOAT_TYPEV2(0.0f);
|
| 440 |
+
}
|
| 441 |
+
|
| 442 |
+
buf_b[buf_ib].qs[iqs * 4 ] = 0;
|
| 443 |
+
buf_b[buf_ib].qs[iqs * 4 + 1] = 0;
|
| 444 |
+
buf_b[buf_ib].qs[iqs * 4 + 2] = 0;
|
| 445 |
+
buf_b[buf_ib].qs[iqs * 4 + 3] = 0;
|
| 446 |
+
}
|
| 447 |
+
}
|
| 448 |
+
|
| 449 |
+
void block_b_to_registers(const uint ib) {
|
| 450 |
+
cache_b.ds = buf_b[ib].ds;
|
| 451 |
+
[[unroll]] for (uint iqs = 0; iqs < BK / 4; iqs++) {
|
| 452 |
+
cache_b.qs[iqs] = buf_b[ib].qs[iqs];
|
| 453 |
+
}
|
| 454 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq_shmem_types.glsl
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#if defined(FA_MMQ_MIXED)
|
| 2 |
+
// Mixed-K flash attention MMQ: superset cache that fits Q4_0/Q4_1/Q5_0/Q5_1/Q8_0.
|
| 3 |
+
// Q4_*/Q5_* only use qs[0..3] and (for Q5_*) qh. Q8_0 uses qs[0..7]. Single-scale
|
| 4 |
+
// types (Q4_0/Q5_0/Q8_0) leave dm.y unused.
|
| 5 |
+
struct block_a_cache {
|
| 6 |
+
int32_t qs[8];
|
| 7 |
+
uint32_t qh;
|
| 8 |
+
FLOAT_TYPEV2 dm;
|
| 9 |
+
};
|
| 10 |
+
#elif defined(DATA_A_Q4_0)
|
| 11 |
+
#define QUANT_R_MMQ 2
|
| 12 |
+
struct block_a_cache {
|
| 13 |
+
uint32_t qs[16/4];
|
| 14 |
+
FLOAT_TYPE dm;
|
| 15 |
+
};
|
| 16 |
+
#elif defined(DATA_A_Q4_1)
|
| 17 |
+
#define QUANT_R_MMQ 2
|
| 18 |
+
struct block_a_cache {
|
| 19 |
+
uint32_t qs[16/4];
|
| 20 |
+
FLOAT_TYPEV2 dm;
|
| 21 |
+
};
|
| 22 |
+
#elif defined(DATA_A_Q5_0)
|
| 23 |
+
#define QUANT_R_MMQ 2
|
| 24 |
+
struct block_a_cache {
|
| 25 |
+
uint32_t qs[16/4];
|
| 26 |
+
uint32_t qh;
|
| 27 |
+
FLOAT_TYPE dm;
|
| 28 |
+
};
|
| 29 |
+
#elif defined(DATA_A_Q5_1)
|
| 30 |
+
#define QUANT_R_MMQ 2
|
| 31 |
+
struct block_a_cache {
|
| 32 |
+
uint32_t qs[16/4];
|
| 33 |
+
uint32_t qh;
|
| 34 |
+
FLOAT_TYPEV2 dm;
|
| 35 |
+
};
|
| 36 |
+
#elif defined(DATA_A_Q8_0)
|
| 37 |
+
#define QUANT_R_MMQ 1
|
| 38 |
+
// AMD likes 4, Intel likes 1 and Nvidia likes 2
|
| 39 |
+
// #define BK_STEP 1
|
| 40 |
+
struct block_a_cache {
|
| 41 |
+
int32_t qs[32/4];
|
| 42 |
+
FLOAT_TYPE dm;
|
| 43 |
+
};
|
| 44 |
+
#elif defined(DATA_A_IQ4_NL)
|
| 45 |
+
#define QUANT_R_MMQ 2
|
| 46 |
+
struct block_a_cache {
|
| 47 |
+
int32_t qs[8];
|
| 48 |
+
FLOAT_TYPE dm;
|
| 49 |
+
};
|
| 50 |
+
#elif defined(DATA_A_MXFP4)
|
| 51 |
+
#define QUANT_R_MMQ 2
|
| 52 |
+
struct block_a_cache {
|
| 53 |
+
int32_t qs[8];
|
| 54 |
+
FLOAT_TYPE d;
|
| 55 |
+
};
|
| 56 |
+
#elif defined(DATA_A_Q2_K)
|
| 57 |
+
#define QUANT_R_MMQ 4
|
| 58 |
+
struct block_a_cache {
|
| 59 |
+
uint32_t qs[2];
|
| 60 |
+
u8vec2 scales;
|
| 61 |
+
FLOAT_TYPEV2 dm;
|
| 62 |
+
};
|
| 63 |
+
#elif defined(DATA_A_Q3_K)
|
| 64 |
+
#define QUANT_R_MMQ 2
|
| 65 |
+
struct block_a_cache {
|
| 66 |
+
uint32_t qs[4];
|
| 67 |
+
FLOAT_TYPEV2 d_scales;
|
| 68 |
+
};
|
| 69 |
+
#elif defined(DATA_A_Q4_K)
|
| 70 |
+
#define QUANT_R_MMQ 2
|
| 71 |
+
struct block_a_cache {
|
| 72 |
+
uint32_t qs[4];
|
| 73 |
+
FLOAT_TYPEV2 dm;
|
| 74 |
+
};
|
| 75 |
+
#elif defined(DATA_A_Q5_K)
|
| 76 |
+
#define QUANT_R_MMQ 1
|
| 77 |
+
struct block_a_cache {
|
| 78 |
+
int32_t qs[8];
|
| 79 |
+
FLOAT_TYPEV2 dm;
|
| 80 |
+
};
|
| 81 |
+
#elif defined(DATA_A_Q6_K)
|
| 82 |
+
#define QUANT_R_MMQ 1
|
| 83 |
+
struct block_a_cache {
|
| 84 |
+
int32_t qs[8];
|
| 85 |
+
FLOAT_TYPEV2 d_scales;
|
| 86 |
+
};
|
| 87 |
+
#endif
|
| 88 |
+
|
| 89 |
+
struct block_b_cache
|
| 90 |
+
{
|
| 91 |
+
int32_t qs[8];
|
| 92 |
+
FLOAT_TYPEV2 ds;
|
| 93 |
+
};
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/multi_add.comp
ADDED
|
@@ -0,0 +1,194 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
|
| 3 |
+
#extension GL_EXT_shader_16bit_storage : require
|
| 4 |
+
#extension GL_EXT_nonuniform_qualifier : enable
|
| 5 |
+
#extension GL_EXT_control_flow_attributes : require
|
| 6 |
+
#if ADD_RMS
|
| 7 |
+
#extension GL_KHR_shader_subgroup_arithmetic : enable
|
| 8 |
+
#extension GL_KHR_shader_subgroup_basic : enable
|
| 9 |
+
#endif
|
| 10 |
+
|
| 11 |
+
#include "types.glsl"
|
| 12 |
+
#include "utils.glsl"
|
| 13 |
+
|
| 14 |
+
layout (push_constant) uniform parameter2
|
| 15 |
+
{
|
| 16 |
+
// shape for dst
|
| 17 |
+
uint ne20; uint ne21; uint ne22; uint ne23;
|
| 18 |
+
|
| 19 |
+
// strides for srcs+dst
|
| 20 |
+
uint nb[12][4];
|
| 21 |
+
|
| 22 |
+
uint rms_partials;
|
| 23 |
+
} p;
|
| 24 |
+
|
| 25 |
+
// No readonly/writeonly decorations. Workaround for MoltenVK Bug, see https://github.com/ggml-org/llama.cpp/issues/15498
|
| 26 |
+
layout (binding = 0) buffer A0 {A_TYPE data_a[];} a0;
|
| 27 |
+
layout (binding = 1) buffer A1 {A_TYPE data_a[];} a1;
|
| 28 |
+
layout (binding = 2) buffer A2 {A_TYPE data_a[];} a2;
|
| 29 |
+
layout (binding = 3) buffer A3 {A_TYPE data_a[];} a3;
|
| 30 |
+
layout (binding = 4) buffer A4 {A_TYPE data_a[];} a4;
|
| 31 |
+
layout (binding = 5) buffer A5 {A_TYPE data_a[];} a5;
|
| 32 |
+
layout (binding = 6) buffer A6 {A_TYPE data_a[];} a6;
|
| 33 |
+
layout (binding = 7) buffer A7 {A_TYPE data_a[];} a7;
|
| 34 |
+
layout (binding = 8) buffer A8 {A_TYPE data_a[];} a8;
|
| 35 |
+
layout (binding = 9) buffer A9 {A_TYPE data_a[];} a9;
|
| 36 |
+
layout (binding = 10) buffer A10 {A_TYPE data_a[];} a10;
|
| 37 |
+
layout (binding = 11) buffer A11 {A_TYPE data_a[];} a11;
|
| 38 |
+
layout (binding = 0) buffer D0 {D_TYPE data_d[];} d0;
|
| 39 |
+
layout (binding = 1) buffer D1 {D_TYPE data_d[];} d1;
|
| 40 |
+
layout (binding = 2) buffer D2 {D_TYPE data_d[];} d2;
|
| 41 |
+
layout (binding = 3) buffer D3 {D_TYPE data_d[];} d3;
|
| 42 |
+
layout (binding = 4) buffer D4 {D_TYPE data_d[];} d4;
|
| 43 |
+
layout (binding = 5) buffer D5 {D_TYPE data_d[];} d5;
|
| 44 |
+
layout (binding = 6) buffer D6 {D_TYPE data_d[];} d6;
|
| 45 |
+
layout (binding = 7) buffer D7 {D_TYPE data_d[];} d7;
|
| 46 |
+
layout (binding = 8) buffer D8 {D_TYPE data_d[];} d8;
|
| 47 |
+
layout (binding = 9) buffer D9 {D_TYPE data_d[];} d9;
|
| 48 |
+
layout (binding = 10) buffer D10 {D_TYPE data_d[];} d10;
|
| 49 |
+
layout (binding = 11) buffer D11 {D_TYPE data_d[];} d11;
|
| 50 |
+
layout (binding = 0, std430) buffer PartialBuf0 {float partial_sums[];} partials0;
|
| 51 |
+
layout (binding = 1, std430) buffer PartialBuf1 {float partial_sums[];} partials1;
|
| 52 |
+
layout (binding = 2, std430) buffer PartialBuf2 {float partial_sums[];} partials2;
|
| 53 |
+
layout (binding = 3, std430) buffer PartialBuf3 {float partial_sums[];} partials3;
|
| 54 |
+
layout (binding = 4, std430) buffer PartialBuf4 {float partial_sums[];} partials4;
|
| 55 |
+
layout (binding = 5, std430) buffer PartialBuf5 {float partial_sums[];} partials5;
|
| 56 |
+
layout (binding = 6, std430) buffer PartialBuf6 {float partial_sums[];} partials6;
|
| 57 |
+
layout (binding = 7, std430) buffer PartialBuf7 {float partial_sums[];} partials7;
|
| 58 |
+
layout (binding = 8, std430) buffer PartialBuf8 {float partial_sums[];} partials8;
|
| 59 |
+
layout (binding = 9, std430) buffer PartialBuf9 {float partial_sums[];} partials9;
|
| 60 |
+
layout (binding = 10, std430) buffer PartialBuf10 {float partial_sums[];} partials10;
|
| 61 |
+
layout (binding = 11, std430) buffer PartialBuf11 {float partial_sums[];} partials11;
|
| 62 |
+
|
| 63 |
+
layout(constant_id = 0) const uint num_srcs = 2;
|
| 64 |
+
|
| 65 |
+
FLOAT_TYPE load_a(uint b, uint i) {
|
| 66 |
+
switch (b) {
|
| 67 |
+
case 0: return FLOAT_TYPE(a0.data_a[i]);
|
| 68 |
+
case 1: return FLOAT_TYPE(a1.data_a[i]);
|
| 69 |
+
case 2: return FLOAT_TYPE(a2.data_a[i]);
|
| 70 |
+
case 3: return FLOAT_TYPE(a3.data_a[i]);
|
| 71 |
+
case 4: return FLOAT_TYPE(a4.data_a[i]);
|
| 72 |
+
case 5: return FLOAT_TYPE(a5.data_a[i]);
|
| 73 |
+
case 6: return FLOAT_TYPE(a6.data_a[i]);
|
| 74 |
+
case 7: return FLOAT_TYPE(a7.data_a[i]);
|
| 75 |
+
case 8: return FLOAT_TYPE(a8.data_a[i]);
|
| 76 |
+
case 9: return FLOAT_TYPE(a9.data_a[i]);
|
| 77 |
+
case 10: return FLOAT_TYPE(a10.data_a[i]);
|
| 78 |
+
case 11: return FLOAT_TYPE(a11.data_a[i]);
|
| 79 |
+
default: return FLOAT_TYPE(0);
|
| 80 |
+
}
|
| 81 |
+
}
|
| 82 |
+
|
| 83 |
+
void store_d(uint b, uint i, FLOAT_TYPE v) {
|
| 84 |
+
switch (b) {
|
| 85 |
+
case 0: d0.data_d[i] = D_TYPE(v); break;
|
| 86 |
+
case 1: d1.data_d[i] = D_TYPE(v); break;
|
| 87 |
+
case 2: d2.data_d[i] = D_TYPE(v); break;
|
| 88 |
+
case 3: d3.data_d[i] = D_TYPE(v); break;
|
| 89 |
+
case 4: d4.data_d[i] = D_TYPE(v); break;
|
| 90 |
+
case 5: d5.data_d[i] = D_TYPE(v); break;
|
| 91 |
+
case 6: d6.data_d[i] = D_TYPE(v); break;
|
| 92 |
+
case 7: d7.data_d[i] = D_TYPE(v); break;
|
| 93 |
+
case 8: d8.data_d[i] = D_TYPE(v); break;
|
| 94 |
+
case 9: d9.data_d[i] = D_TYPE(v); break;
|
| 95 |
+
case 10: d10.data_d[i] = D_TYPE(v); break;
|
| 96 |
+
case 11: d11.data_d[i] = D_TYPE(v); break;
|
| 97 |
+
default: break;
|
| 98 |
+
}
|
| 99 |
+
}
|
| 100 |
+
|
| 101 |
+
void store_partial(uint b, uint i, float v) {
|
| 102 |
+
switch (b) {
|
| 103 |
+
case 0: partials0.partial_sums[i] = v; break;
|
| 104 |
+
case 1: partials1.partial_sums[i] = v; break;
|
| 105 |
+
case 2: partials2.partial_sums[i] = v; break;
|
| 106 |
+
case 3: partials3.partial_sums[i] = v; break;
|
| 107 |
+
case 4: partials4.partial_sums[i] = v; break;
|
| 108 |
+
case 5: partials5.partial_sums[i] = v; break;
|
| 109 |
+
case 6: partials6.partial_sums[i] = v; break;
|
| 110 |
+
case 7: partials7.partial_sums[i] = v; break;
|
| 111 |
+
case 8: partials8.partial_sums[i] = v; break;
|
| 112 |
+
case 9: partials9.partial_sums[i] = v; break;
|
| 113 |
+
case 10: partials10.partial_sums[i] = v; break;
|
| 114 |
+
case 11: partials11.partial_sums[i] = v; break;
|
| 115 |
+
default: break;
|
| 116 |
+
}
|
| 117 |
+
}
|
| 118 |
+
|
| 119 |
+
uint src_idx(uint s, uint i00, uint i01, uint i02, uint i03) {
|
| 120 |
+
return i03*p.nb[s][3] + i02*p.nb[s][2] + i01*p.nb[s][1] + i00*p.nb[s][0];
|
| 121 |
+
}
|
| 122 |
+
|
| 123 |
+
uint dst_idx(uint i00, uint i01, uint i02, uint i03) {
|
| 124 |
+
uint nb20 = p.nb[num_srcs][0];
|
| 125 |
+
uint nb21 = p.nb[num_srcs][1];
|
| 126 |
+
uint nb22 = p.nb[num_srcs][2];
|
| 127 |
+
uint nb23 = p.nb[num_srcs][3];
|
| 128 |
+
return i03*nb23 + i02*nb22 + i01*nb21 + i00*nb20;
|
| 129 |
+
}
|
| 130 |
+
|
| 131 |
+
uint get_idx() {
|
| 132 |
+
return gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;
|
| 133 |
+
}
|
| 134 |
+
|
| 135 |
+
const uint num_threads = 256;
|
| 136 |
+
|
| 137 |
+
layout(local_size_x = num_threads, local_size_y = 1, local_size_z = 1) in;
|
| 138 |
+
|
| 139 |
+
#if ADD_RMS
|
| 140 |
+
// XXX TODO this could be sized based on number of subgroups, but that't not considered a constant
|
| 141 |
+
shared FLOAT_TYPE sumsh[num_threads];
|
| 142 |
+
#endif
|
| 143 |
+
|
| 144 |
+
void main() {
|
| 145 |
+
uint idx = get_idx();
|
| 146 |
+
uint orig_idx = idx;
|
| 147 |
+
|
| 148 |
+
uint ne = p.ne20 * p.ne21 * p.ne22 * p.ne23;
|
| 149 |
+
|
| 150 |
+
// num_threads * num_iter must equal 512, to match the wg_denoms and get_idx calculation
|
| 151 |
+
const uint num_iter = 2;
|
| 152 |
+
|
| 153 |
+
FLOAT_TYPE sum_sq = 0;
|
| 154 |
+
|
| 155 |
+
[[unroll]] for (uint i = 0; i < num_iter; ++i) {
|
| 156 |
+
if (idx >= ne) {
|
| 157 |
+
continue;
|
| 158 |
+
}
|
| 159 |
+
uint i00, i01, i02, i03;
|
| 160 |
+
get_indices(idx, i00, i01, i02, i03, p.ne20, p.ne21, p.ne22, p.ne23);
|
| 161 |
+
|
| 162 |
+
FLOAT_TYPE sum = FLOAT_TYPE(0);
|
| 163 |
+
[[unroll]] for (uint s = 0; s < num_srcs; ++s) {
|
| 164 |
+
sum += load_a(s, src_idx(s, i00, i01, i02, i03));
|
| 165 |
+
}
|
| 166 |
+
sum_sq += sum*sum;
|
| 167 |
+
store_d(num_srcs, dst_idx(i00, i01, i02, i03), sum);
|
| 168 |
+
|
| 169 |
+
idx += num_threads;
|
| 170 |
+
}
|
| 171 |
+
|
| 172 |
+
#if ADD_RMS
|
| 173 |
+
if (p.rms_partials != 0) {
|
| 174 |
+
// reduce the sum within each subgroup, then across subgroups
|
| 175 |
+
const uint NumSubgroups = num_threads / gl_SubgroupSize;
|
| 176 |
+
sum_sq = subgroupAdd(sum_sq);
|
| 177 |
+
if (gl_SubgroupInvocationID == 0) {
|
| 178 |
+
sumsh[gl_SubgroupID] = sum_sq;
|
| 179 |
+
}
|
| 180 |
+
barrier();
|
| 181 |
+
[[unroll]] for (uint s = NumSubgroups / 2; s > 0; s >>= 1) {
|
| 182 |
+
if (gl_SubgroupID < s && gl_SubgroupInvocationID == 0) {
|
| 183 |
+
sum_sq += sumsh[gl_SubgroupID + s];
|
| 184 |
+
sumsh[gl_SubgroupID] = sum_sq;
|
| 185 |
+
}
|
| 186 |
+
barrier();
|
| 187 |
+
}
|
| 188 |
+
|
| 189 |
+
if (gl_SubgroupID == 0 && gl_SubgroupInvocationID == 0) {
|
| 190 |
+
store_partial(num_srcs + 1, orig_idx / (num_iter * num_threads), sum_sq);
|
| 191 |
+
}
|
| 192 |
+
}
|
| 193 |
+
#endif
|
| 194 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/norm.comp
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
|
| 3 |
+
#include "types.glsl"
|
| 4 |
+
#include "generic_unary_head.glsl"
|
| 5 |
+
|
| 6 |
+
#extension GL_EXT_control_flow_attributes : enable
|
| 7 |
+
#define BLOCK_SIZE 512
|
| 8 |
+
|
| 9 |
+
layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in;
|
| 10 |
+
|
| 11 |
+
shared vec2 sum[BLOCK_SIZE];
|
| 12 |
+
|
| 13 |
+
void main() {
|
| 14 |
+
const uint row = gl_WorkGroupID.z * 262144 + gl_WorkGroupID.y * 512 + gl_WorkGroupID.x;
|
| 15 |
+
const uint tid = gl_LocalInvocationID.x;
|
| 16 |
+
|
| 17 |
+
const uint a_base = get_aoffset() + src0_idx(row * p.ne00);
|
| 18 |
+
const uint d_base = get_doffset() + dst_idx(row * p.ne10);
|
| 19 |
+
|
| 20 |
+
sum[tid] = vec2(0.0f, 0.0f);
|
| 21 |
+
|
| 22 |
+
[[unroll]] for (uint i0 = tid; i0 < p.ne00; i0 += BLOCK_SIZE) {
|
| 23 |
+
const float xi = float(data_a[a_base + i0*p.nb00]);
|
| 24 |
+
sum[tid].x += xi;
|
| 25 |
+
sum[tid].y += xi * xi;
|
| 26 |
+
}
|
| 27 |
+
|
| 28 |
+
// sum up partial sums and write back result
|
| 29 |
+
barrier();
|
| 30 |
+
[[unroll]] for (int s = BLOCK_SIZE / 2; s > 0; s >>= 1) {
|
| 31 |
+
if (tid < s) {
|
| 32 |
+
sum[tid] += sum[tid + s];
|
| 33 |
+
}
|
| 34 |
+
barrier();
|
| 35 |
+
}
|
| 36 |
+
|
| 37 |
+
const float mean = sum[0].x / p.ne00;
|
| 38 |
+
const float var = sum[0].y / p.ne00 - mean * mean;
|
| 39 |
+
const float inv_std = inversesqrt(var + p.param1);
|
| 40 |
+
|
| 41 |
+
[[unroll]] for (uint i0 = tid; i0 < p.ne00; i0 += BLOCK_SIZE) {
|
| 42 |
+
data_d[d_base + i0*p.nb10] = D_TYPE((float(data_a[a_base + i0*p.nb00]) - mean) * inv_std);
|
| 43 |
+
}
|
| 44 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_adamw.comp
ADDED
|
@@ -0,0 +1,42 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
|
| 3 |
+
#include "generic_head.glsl"
|
| 4 |
+
#include "types.glsl"
|
| 5 |
+
|
| 6 |
+
#extension GL_EXT_control_flow_attributes : enable
|
| 7 |
+
|
| 8 |
+
layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
|
| 9 |
+
|
| 10 |
+
layout (binding = 0) buffer X {A_TYPE x[];};
|
| 11 |
+
layout (binding = 1) readonly buffer G {A_TYPE grad[];};
|
| 12 |
+
layout (binding = 2) buffer GM {A_TYPE gradm[];};
|
| 13 |
+
layout (binding = 3) buffer GV {A_TYPE gradv[];};
|
| 14 |
+
layout (binding = 4) readonly buffer P {float params[7];};
|
| 15 |
+
|
| 16 |
+
void main() {
|
| 17 |
+
const uint i = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;
|
| 18 |
+
|
| 19 |
+
if (i >= p.KX) {
|
| 20 |
+
return;
|
| 21 |
+
}
|
| 22 |
+
|
| 23 |
+
const float alpha = params[0];
|
| 24 |
+
const float beta1 = params[1];
|
| 25 |
+
const float beta2 = params[2];
|
| 26 |
+
const float eps = params[3];
|
| 27 |
+
const float wd = params[4];
|
| 28 |
+
const float beta1h = params[5];
|
| 29 |
+
const float beta2h = params[6];
|
| 30 |
+
|
| 31 |
+
const float gi = grad[i];
|
| 32 |
+
const float gmi = gradm[i]*beta1 + gi*(1.0f - beta1);
|
| 33 |
+
const float gvi = gradv[i]*beta2 + gi*gi*(1.0f - beta2);
|
| 34 |
+
|
| 35 |
+
gradm[i] = gmi;
|
| 36 |
+
gradv[i] = gvi;
|
| 37 |
+
|
| 38 |
+
const float mh = gmi*beta1h;
|
| 39 |
+
const float vh = sqrt(gvi*beta2h) + eps;
|
| 40 |
+
|
| 41 |
+
x[i] = x[i]*(1.0f - alpha*wd) - alpha*mh/vh;
|
| 42 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/opt_step_sgd.comp
ADDED
|
@@ -0,0 +1,22 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
|
| 3 |
+
#include "generic_head.glsl"
|
| 4 |
+
|
| 5 |
+
layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
|
| 6 |
+
|
| 7 |
+
layout (binding = 0) buffer X {A_TYPE data_x[];};
|
| 8 |
+
layout (binding = 1) readonly buffer G {A_TYPE data_grad[];};
|
| 9 |
+
layout (binding = 2) readonly buffer P {float data_params[2];};
|
| 10 |
+
|
| 11 |
+
void main() {
|
| 12 |
+
const uint i = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;
|
| 13 |
+
|
| 14 |
+
if (i >= p.KX) {
|
| 15 |
+
return;
|
| 16 |
+
}
|
| 17 |
+
|
| 18 |
+
const float alpha = data_params[0];
|
| 19 |
+
const float keep = 1.f - alpha * data_params[1];
|
| 20 |
+
|
| 21 |
+
data_x[i] = data_x[i] * keep - alpha * data_grad[i];
|
| 22 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/pad.comp
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
|
| 3 |
+
#include "types.glsl"
|
| 4 |
+
|
| 5 |
+
layout (push_constant) uniform parameter
|
| 6 |
+
{
|
| 7 |
+
uint ne;
|
| 8 |
+
uint ne00; uint ne01; uint ne02; uint ne03; uint nb00; uint nb01; uint nb02; uint nb03;
|
| 9 |
+
uint ne10; uint ne11; uint ne12; uint ne13; uint nb10; uint nb11; uint nb12; uint nb13;
|
| 10 |
+
uint misalign_offsets;
|
| 11 |
+
uint circular;
|
| 12 |
+
|
| 13 |
+
uint lp0; uint rp0;
|
| 14 |
+
uint lp1; uint rp1;
|
| 15 |
+
uint lp2; uint rp2;
|
| 16 |
+
uint lp3; uint rp3;
|
| 17 |
+
} p;
|
| 18 |
+
|
| 19 |
+
uint get_aoffset() { return p.misalign_offsets >> 16; }
|
| 20 |
+
uint get_doffset() { return p.misalign_offsets & 0xFFFF; }
|
| 21 |
+
|
| 22 |
+
uint wrap_around(int coord, uint size) {
|
| 23 |
+
return (uint(coord + int(size))) % size; // add size to avoid issues with negative
|
| 24 |
+
}
|
| 25 |
+
|
| 26 |
+
layout (binding = 0) readonly buffer A {A_TYPE data_a[];};
|
| 27 |
+
layout (binding = 1) writeonly buffer D {D_TYPE data_d[];};
|
| 28 |
+
|
| 29 |
+
layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
|
| 30 |
+
|
| 31 |
+
void main() {
|
| 32 |
+
const uint idx = gl_GlobalInvocationID.z * 262144 + gl_GlobalInvocationID.y * 512 + gl_GlobalInvocationID.x;
|
| 33 |
+
|
| 34 |
+
if (idx >= p.ne) {
|
| 35 |
+
return;
|
| 36 |
+
}
|
| 37 |
+
|
| 38 |
+
const uint i3 = idx / (p.ne12*p.ne11*p.ne10);
|
| 39 |
+
const uint i3_offset = i3 * p.ne12*p.ne11*p.ne10;
|
| 40 |
+
const uint i2 = (idx - i3_offset) / (p.ne11*p.ne10);
|
| 41 |
+
const uint i2_offset = i2*p.ne11*p.ne10;
|
| 42 |
+
const uint i1 = (idx - i3_offset - i2_offset) / p.ne10;
|
| 43 |
+
const uint i0 = idx - i3_offset - i2_offset - i1*p.ne10;
|
| 44 |
+
|
| 45 |
+
const uint src0_idx = (i3 - p.lp3)*p.nb03 + (i2 - p.lp2)*p.nb02 + (i1 - p.lp1)*p.nb01 + (i0 - p.lp0)*p.nb00;
|
| 46 |
+
const uint dst_idx = i3*p.nb13 + i2*p.nb12 + i1*p.nb11 + i0*p.nb10;
|
| 47 |
+
|
| 48 |
+
if (p.circular != 0u) {
|
| 49 |
+
const uint ci0 = wrap_around(int(i0) - int(p.lp0), p.ne00);
|
| 50 |
+
const uint ci1 = wrap_around(int(i1) - int(p.lp1), p.ne01);
|
| 51 |
+
const uint ci2 = wrap_around(int(i2) - int(p.lp2), p.ne02);
|
| 52 |
+
const uint ci3 = wrap_around(int(i3) - int(p.lp3), p.ne03);
|
| 53 |
+
const uint circular_src_idx = ci3*p.nb03 + ci2*p.nb02 + ci1*p.nb01 + ci0*p.nb00;
|
| 54 |
+
data_d[get_doffset() + dst_idx] = D_TYPE(data_a[get_aoffset() + circular_src_idx]);
|
| 55 |
+
} else {
|
| 56 |
+
const bool is_src0 = i0 >= p.lp0 && i0 < p.ne10 - p.rp0 &&
|
| 57 |
+
i1 >= p.lp1 && i1 < p.ne11 - p.rp1 &&
|
| 58 |
+
i2 >= p.lp2 && i2 < p.ne12 - p.rp2 &&
|
| 59 |
+
i3 >= p.lp3 && i3 < p.ne13 - p.rp3;
|
| 60 |
+
data_d[get_doffset() + dst_idx] = D_TYPE(is_src0 ? data_a[get_aoffset() + src0_idx] : 0.0f);
|
| 61 |
+
}
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/pool2d.comp
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
|
| 3 |
+
#include "types.glsl"
|
| 4 |
+
|
| 5 |
+
#extension GL_EXT_shader_16bit_storage : require
|
| 6 |
+
|
| 7 |
+
layout(push_constant) uniform parameter {
|
| 8 |
+
uint IW; uint IH;
|
| 9 |
+
uint OW; uint OH;
|
| 10 |
+
uint OC;
|
| 11 |
+
uint pelements;
|
| 12 |
+
uint op;
|
| 13 |
+
int k0; int k1;
|
| 14 |
+
int s0; int s1;
|
| 15 |
+
int p0; int p1;
|
| 16 |
+
} p;
|
| 17 |
+
|
| 18 |
+
#define BLOCK_SIZE 512
|
| 19 |
+
#define FLT_MAX 3.402823466e+38F
|
| 20 |
+
#define OP_POOL_MAX 0u
|
| 21 |
+
#define OP_POOL_AVG 1u
|
| 22 |
+
|
| 23 |
+
layout (local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in;
|
| 24 |
+
|
| 25 |
+
layout(binding = 0) readonly buffer X {A_TYPE data_a[];};
|
| 26 |
+
layout(binding = 1) writeonly buffer D {D_TYPE data_d[];};
|
| 27 |
+
|
| 28 |
+
void main() {
|
| 29 |
+
const uint idx = gl_GlobalInvocationID.x;
|
| 30 |
+
if (idx >= p.pelements) {
|
| 31 |
+
return;
|
| 32 |
+
}
|
| 33 |
+
|
| 34 |
+
const uint O_HW = p.OW * p.OH;
|
| 35 |
+
|
| 36 |
+
const uint nc = idx / O_HW;
|
| 37 |
+
const uint cur_oh = (idx % O_HW) / p.OW;
|
| 38 |
+
const uint cur_ow = (idx % O_HW) % p.OW;
|
| 39 |
+
|
| 40 |
+
const int start_h = int(cur_oh) * p.s0 - p.p0;
|
| 41 |
+
const uint bh = max(start_h, 0);
|
| 42 |
+
const uint eh = min(start_h + p.k0, p.IH);
|
| 43 |
+
|
| 44 |
+
const int start_w = int(cur_ow) * p.s1 - p.p1;
|
| 45 |
+
const uint bw = max(start_w, 0);
|
| 46 |
+
const uint ew = min(start_w + p.k1, p.IW);
|
| 47 |
+
|
| 48 |
+
const float scale = 1.0 / float(p.k0 * p.k1);
|
| 49 |
+
float res;
|
| 50 |
+
|
| 51 |
+
if (p.op == OP_POOL_AVG) {
|
| 52 |
+
res = 0.0;
|
| 53 |
+
} else if (p.op == OP_POOL_MAX) {
|
| 54 |
+
res = -FLT_MAX;
|
| 55 |
+
} else {
|
| 56 |
+
return;
|
| 57 |
+
}
|
| 58 |
+
|
| 59 |
+
#pragma unroll
|
| 60 |
+
for (uint i = bh; i < eh; i++) {
|
| 61 |
+
#pragma unroll
|
| 62 |
+
for (uint j = bw; j < ew; j++) {
|
| 63 |
+
const float cur = D_TYPE(data_a[nc * p.IH * p.IW + i * p.IW + j]);
|
| 64 |
+
|
| 65 |
+
if (p.op == OP_POOL_AVG) {
|
| 66 |
+
res += cur * scale;
|
| 67 |
+
} else if (p.op == OP_POOL_MAX) {
|
| 68 |
+
res = max(res, cur);
|
| 69 |
+
}
|
| 70 |
+
}
|
| 71 |
+
}
|
| 72 |
+
|
| 73 |
+
data_d[nc * O_HW + cur_oh * p.OW + cur_ow] = res;
|
| 74 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/quantize_q8_1.comp
ADDED
|
@@ -0,0 +1,127 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
|
| 3 |
+
#extension GL_EXT_control_flow_attributes : require
|
| 4 |
+
#extension GL_EXT_shader_16bit_storage : require
|
| 5 |
+
|
| 6 |
+
#ifdef USE_SUBGROUPS
|
| 7 |
+
#extension GL_KHR_shader_subgroup_basic : require
|
| 8 |
+
#extension GL_KHR_shader_subgroup_clustered : require
|
| 9 |
+
|
| 10 |
+
#define INVOCATION_ID gl_SubgroupInvocationID.x
|
| 11 |
+
#else
|
| 12 |
+
#define INVOCATION_ID gl_LocalInvocationID.x
|
| 13 |
+
#endif
|
| 14 |
+
|
| 15 |
+
layout (push_constant) uniform parameter
|
| 16 |
+
{
|
| 17 |
+
uint ne;
|
| 18 |
+
uint num_blocks;
|
| 19 |
+
} p;
|
| 20 |
+
|
| 21 |
+
#include "types.glsl"
|
| 22 |
+
|
| 23 |
+
layout(constant_id = 0) const uint GROUP_SIZE = 32;
|
| 24 |
+
layout(local_size_x_id = 0, local_size_y = 1, local_size_z = 1) in;
|
| 25 |
+
|
| 26 |
+
layout (binding = 0) readonly buffer A {vec4 data_a[];};
|
| 27 |
+
#ifndef QBLOCK_X4
|
| 28 |
+
layout (binding = 1) writeonly buffer D {block_q8_1_packed32 data_b[];};
|
| 29 |
+
#else
|
| 30 |
+
layout (binding = 1) writeonly buffer D {block_q8_1_x4 data_b[];};
|
| 31 |
+
#endif
|
| 32 |
+
|
| 33 |
+
#ifndef USE_SUBGROUPS
|
| 34 |
+
shared float shmem[GROUP_SIZE];
|
| 35 |
+
#endif
|
| 36 |
+
|
| 37 |
+
void quantize(const uint wgid) {
|
| 38 |
+
const uint tid = INVOCATION_ID;
|
| 39 |
+
|
| 40 |
+
// Each thread handles a vec4, so 8 threads handle a block
|
| 41 |
+
const uint blocks_per_group = GROUP_SIZE / 8;
|
| 42 |
+
|
| 43 |
+
const uint block_in_wg = tid / 8;
|
| 44 |
+
|
| 45 |
+
const uint ib = wgid * blocks_per_group + block_in_wg;
|
| 46 |
+
const uint iqs = tid % 8;
|
| 47 |
+
|
| 48 |
+
#ifdef QBLOCK_X4
|
| 49 |
+
const uint ibx4_outer = ib / 4;
|
| 50 |
+
const uint ibx4_inner = ib % 4;
|
| 51 |
+
|
| 52 |
+
const uint required_x4_blocks = (p.ne + 127) / 128;
|
| 53 |
+
if (ibx4_outer >= required_x4_blocks) {
|
| 54 |
+
return;
|
| 55 |
+
}
|
| 56 |
+
#endif
|
| 57 |
+
|
| 58 |
+
const uint a_idx = ib * 8 + iqs;
|
| 59 |
+
|
| 60 |
+
vec4 vals = a_idx < p.ne / 4 ? data_a[a_idx] : vec4(0.0f);
|
| 61 |
+
const vec4 abs_vals = abs(vals);
|
| 62 |
+
|
| 63 |
+
// Find absolute max for each block
|
| 64 |
+
const float thread_max = max(max(abs_vals.x, abs_vals.y), max(abs_vals.z, abs_vals.w));
|
| 65 |
+
#ifndef USE_SUBGROUPS
|
| 66 |
+
shmem[tid] = thread_max;
|
| 67 |
+
barrier();
|
| 68 |
+
[[unroll]] for (uint s = 4; s > 0; s >>= 1) {
|
| 69 |
+
if (iqs < s) {
|
| 70 |
+
shmem[tid] = max(shmem[tid], shmem[tid + s]);
|
| 71 |
+
}
|
| 72 |
+
barrier();
|
| 73 |
+
}
|
| 74 |
+
|
| 75 |
+
const float amax = shmem[block_in_wg * 8];
|
| 76 |
+
#else
|
| 77 |
+
const float amax = subgroupClusteredMax(thread_max, 8);
|
| 78 |
+
#endif
|
| 79 |
+
|
| 80 |
+
const float d = amax / 127.0;
|
| 81 |
+
const float d_inv = d != 0.0 ? 1.0 / d : 0.0;
|
| 82 |
+
vals = round(vals * d_inv);
|
| 83 |
+
|
| 84 |
+
#ifndef QBLOCK_X4
|
| 85 |
+
data_b[ib].qs[iqs] = pack32(i8vec4(round(vals)));
|
| 86 |
+
#else
|
| 87 |
+
data_b[ibx4_outer].qs[ibx4_inner * 8 + iqs] = pack32(i8vec4(round(vals)));
|
| 88 |
+
#endif
|
| 89 |
+
|
| 90 |
+
#ifndef USE_SUBGROUPS
|
| 91 |
+
barrier();
|
| 92 |
+
#endif
|
| 93 |
+
|
| 94 |
+
// Calculate the sum for each block
|
| 95 |
+
const float thread_sum = vals.x + vals.y + vals.z + vals.w;
|
| 96 |
+
#ifndef USE_SUBGROUPS
|
| 97 |
+
shmem[tid] = thread_sum;
|
| 98 |
+
barrier();
|
| 99 |
+
[[unroll]] for (uint s = 4; s > 0; s >>= 1) {
|
| 100 |
+
if (iqs < s) {
|
| 101 |
+
shmem[tid] += shmem[tid + s];
|
| 102 |
+
}
|
| 103 |
+
barrier();
|
| 104 |
+
}
|
| 105 |
+
#else
|
| 106 |
+
const float sum = subgroupClusteredAdd(thread_sum, 8);
|
| 107 |
+
#endif
|
| 108 |
+
if (iqs == 0) {
|
| 109 |
+
#ifndef USE_SUBGROUPS
|
| 110 |
+
const float sum = shmem[tid];
|
| 111 |
+
#endif
|
| 112 |
+
|
| 113 |
+
#ifndef QBLOCK_X4
|
| 114 |
+
data_b[ib].ds = f16vec2(vec2(d, sum * d));
|
| 115 |
+
#else
|
| 116 |
+
data_b[ibx4_outer].ds[ibx4_inner] = f16vec2(vec2(d, sum * d));
|
| 117 |
+
#endif
|
| 118 |
+
}
|
| 119 |
+
}
|
| 120 |
+
|
| 121 |
+
void main() {
|
| 122 |
+
uint wgid = gl_WorkGroupID.x;
|
| 123 |
+
while (wgid < p.num_blocks) {
|
| 124 |
+
quantize(wgid);
|
| 125 |
+
wgid += gl_NumWorkGroups.x;
|
| 126 |
+
}
|
| 127 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/reglu.comp
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
|
| 3 |
+
#include "glu_head.glsl"
|
| 4 |
+
|
| 5 |
+
float op(float a, float b) {
|
| 6 |
+
return max(a, 0.0f) * b;
|
| 7 |
+
}
|
| 8 |
+
|
| 9 |
+
#include "glu_main.glsl"
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/repeat.comp
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
|
| 3 |
+
#include "types.glsl"
|
| 4 |
+
#include "generic_unary_head.glsl"
|
| 5 |
+
|
| 6 |
+
layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
|
| 7 |
+
|
| 8 |
+
uint src0_idx_mod(uint idx) {
|
| 9 |
+
const uint i13 = idx / (p.ne12*p.ne11*p.ne10);
|
| 10 |
+
const uint i13_offset = i13 * p.ne12*p.ne11*p.ne10;
|
| 11 |
+
const uint i12 = (idx - i13_offset) / (p.ne11*p.ne10);
|
| 12 |
+
const uint i12_offset = i12*p.ne11*p.ne10;
|
| 13 |
+
const uint i11 = (idx - i13_offset - i12_offset) / p.ne10;
|
| 14 |
+
const uint i10 = idx - i13_offset - i12_offset - i11*p.ne10;
|
| 15 |
+
return (i13 % p.ne03)*p.nb03 + (i12 % p.ne02)*p.nb02 + (i11 % p.ne01)*p.nb01 + (i10 % p.ne00)*p.nb00;
|
| 16 |
+
}
|
| 17 |
+
|
| 18 |
+
void main() {
|
| 19 |
+
const uint idx = get_idx();
|
| 20 |
+
|
| 21 |
+
if (idx >= p.ne) {
|
| 22 |
+
return;
|
| 23 |
+
}
|
| 24 |
+
|
| 25 |
+
data_d[get_doffset() + dst_idx(idx)] = D_TYPE(data_a[get_aoffset() + src0_idx_mod(idx)]);
|
| 26 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/repeat_back.comp
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
|
| 3 |
+
#include "types.glsl"
|
| 4 |
+
#include "generic_unary_head.glsl"
|
| 5 |
+
|
| 6 |
+
layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
|
| 7 |
+
|
| 8 |
+
void main() {
|
| 9 |
+
const uint idx = get_idx();
|
| 10 |
+
|
| 11 |
+
if (idx >= p.ne) {
|
| 12 |
+
return;
|
| 13 |
+
}
|
| 14 |
+
|
| 15 |
+
// Destination multi-index (inlined dst_idx)
|
| 16 |
+
const uint i13 = fastdiv(idx, p.ne1_012mp, fastdiv_L(p.ne1_Ls, 0));
|
| 17 |
+
const uint i13_offset = i13 * p.ne12*p.ne11*p.ne10;
|
| 18 |
+
const uint i12 = fastdiv(idx - i13_offset, p.ne1_01mp, fastdiv_L(p.ne1_Ls, 1));
|
| 19 |
+
const uint i12_offset = i12*p.ne11*p.ne10;
|
| 20 |
+
const uint i11 = fastdiv(idx - i13_offset - i12_offset, p.ne1_0mp, fastdiv_L(p.ne1_Ls, 2));
|
| 21 |
+
const uint i10 = idx - i13_offset - i12_offset - i11*p.ne10;
|
| 22 |
+
const uint d_idx = i13*p.nb13 + i12*p.nb12 + i11*p.nb11 + i10*p.nb10;
|
| 23 |
+
|
| 24 |
+
// Accumulate from sources
|
| 25 |
+
A_TYPE acc = A_TYPE(0);
|
| 26 |
+
for (uint i3 = i13; i3 < p.ne03; i3 += p.ne13) {
|
| 27 |
+
for (uint i2 = i12; i2 < p.ne02; i2 += p.ne12) {
|
| 28 |
+
for (uint i1 = i11; i1 < p.ne01; i1 += p.ne11) {
|
| 29 |
+
for (uint i0 = i10; i0 < p.ne00; i0 += p.ne10) {
|
| 30 |
+
acc += data_a[i3*p.nb03 + i2*p.nb02 + i1*p.nb01 + i0*p.nb00];
|
| 31 |
+
}
|
| 32 |
+
}
|
| 33 |
+
}
|
| 34 |
+
}
|
| 35 |
+
|
| 36 |
+
data_d[get_doffset() + d_idx] = D_TYPE(acc);
|
| 37 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm.comp
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
|
| 3 |
+
#include "generic_binary_head.glsl"
|
| 4 |
+
#include "types.glsl"
|
| 5 |
+
|
| 6 |
+
#if RMS_NORM_ROPE_FUSION
|
| 7 |
+
|
| 8 |
+
layout (binding = 0) readonly buffer A {A_TYPE data_a[];};
|
| 9 |
+
layout (binding = 1) readonly buffer B {B_TYPE data_b[];};
|
| 10 |
+
|
| 11 |
+
// data is passed from rms_norm -> rope through shared memory.
|
| 12 |
+
// rms_norm calls this data_d, rope calls this rope_data_a.
|
| 13 |
+
// Binding 2 is not used
|
| 14 |
+
shared FLOAT_TYPE rope_data_a[1024];
|
| 15 |
+
#define data_d rope_data_a
|
| 16 |
+
|
| 17 |
+
layout (binding = 3) readonly buffer R_Y {int rope_data_pos[];};
|
| 18 |
+
layout (binding = 4) readonly buffer R_Z {float rope_data_ff[];};
|
| 19 |
+
layout (binding = 5) writeonly buffer R_D {ROPE_D_TYPE rope_data_d[];};
|
| 20 |
+
layout (binding = 6) readonly buffer R_I {uvec2 rope_data_i[];}; // indices for set_rows
|
| 21 |
+
|
| 22 |
+
#include "rope_params.glsl"
|
| 23 |
+
#include "rope_funcs.glsl"
|
| 24 |
+
|
| 25 |
+
#define GGML_ROPE_TYPE_NORMAL 0
|
| 26 |
+
#define GGML_ROPE_TYPE_NEOX 2
|
| 27 |
+
#define GGML_ROPE_TYPE_MROPE 8
|
| 28 |
+
#define GGML_ROPE_TYPE_VISION 24
|
| 29 |
+
|
| 30 |
+
#endif
|
| 31 |
+
|
| 32 |
+
#extension GL_EXT_control_flow_attributes : enable
|
| 33 |
+
#define BLOCK_SIZE 512
|
| 34 |
+
|
| 35 |
+
layout (constant_id = 1) const bool do_multiply = false;
|
| 36 |
+
|
| 37 |
+
layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in;
|
| 38 |
+
|
| 39 |
+
shared FLOAT_TYPE sumsh[BLOCK_SIZE];
|
| 40 |
+
|
| 41 |
+
void rms_norm(uint num_iters) {
|
| 42 |
+
const uint ncols = p.ne00;
|
| 43 |
+
const uint nrows = gl_NumWorkGroups.x;
|
| 44 |
+
const uint nchannels = gl_NumWorkGroups.y;
|
| 45 |
+
|
| 46 |
+
const uint row = gl_WorkGroupID.x;
|
| 47 |
+
const uint channel = gl_WorkGroupID.y;
|
| 48 |
+
const uint samp = gl_WorkGroupID.z;
|
| 49 |
+
const uint tid = gl_LocalInvocationID.x;
|
| 50 |
+
|
| 51 |
+
const uint stride_row = p.nb01;
|
| 52 |
+
const uint stride_channel = p.nb02;
|
| 53 |
+
const uint stride_sample = p.nb03;
|
| 54 |
+
|
| 55 |
+
uint32_t a_offset = samp*stride_sample + channel*stride_channel + row*stride_row + get_aoffset();
|
| 56 |
+
uint32_t b_offset = src1_idx(0, row, channel, samp) + get_boffset();
|
| 57 |
+
#if RMS_NORM_ROPE_FUSION
|
| 58 |
+
// Per-row offset in shared memory
|
| 59 |
+
uint32_t d_offset = 0;
|
| 60 |
+
#else
|
| 61 |
+
uint32_t d_offset = ((samp*nchannels + channel)*nrows + row)*ncols + get_doffset();
|
| 62 |
+
#endif
|
| 63 |
+
FLOAT_TYPE sum = FLOAT_TYPE(0.0f); // partial sum for thread in warp
|
| 64 |
+
|
| 65 |
+
[[unroll]] for (uint col = tid, idx = 0; idx < num_iters; col += BLOCK_SIZE, ++idx) {
|
| 66 |
+
FLOAT_TYPE xi = FLOAT_TYPE(0);
|
| 67 |
+
if (col < ncols) {
|
| 68 |
+
xi = FLOAT_TYPE(data_a[a_offset + col]);
|
| 69 |
+
}
|
| 70 |
+
sum += xi * xi;
|
| 71 |
+
}
|
| 72 |
+
|
| 73 |
+
sumsh[tid] = sum;
|
| 74 |
+
// sum up partial sums and write back result
|
| 75 |
+
barrier();
|
| 76 |
+
[[unroll]] for (int s = BLOCK_SIZE / 2; s > 0; s >>= 1) {
|
| 77 |
+
if (tid < s) {
|
| 78 |
+
sum += sumsh[tid + s];
|
| 79 |
+
sumsh[tid] = sum;
|
| 80 |
+
}
|
| 81 |
+
barrier();
|
| 82 |
+
}
|
| 83 |
+
sum = sumsh[0];
|
| 84 |
+
|
| 85 |
+
const FLOAT_TYPE mean = sum / FLOAT_TYPE(ncols);
|
| 86 |
+
const FLOAT_TYPE scale = inversesqrt(mean + FLOAT_TYPE(p.param1));
|
| 87 |
+
|
| 88 |
+
if (do_multiply) {
|
| 89 |
+
if (ncols > p.ne10) {
|
| 90 |
+
[[unroll]] for (uint col = tid, idx = 0; idx < num_iters; col += BLOCK_SIZE, ++idx) {
|
| 91 |
+
if (col >= ncols) {
|
| 92 |
+
continue;
|
| 93 |
+
}
|
| 94 |
+
data_d[d_offset + col] = D_TYPE(scale * FLOAT_TYPE(data_a[a_offset + col]) * FLOAT_TYPE(data_b[b_offset + fastmod(col, p.ne10)]));
|
| 95 |
+
}
|
| 96 |
+
} else {
|
| 97 |
+
[[unroll]] for (uint col = tid, idx = 0; idx < num_iters; col += BLOCK_SIZE, ++idx) {
|
| 98 |
+
if (col >= ncols) {
|
| 99 |
+
continue;
|
| 100 |
+
}
|
| 101 |
+
data_d[d_offset + col] = D_TYPE(scale * FLOAT_TYPE(data_a[a_offset + col]) * FLOAT_TYPE(data_b[b_offset + col]));
|
| 102 |
+
}
|
| 103 |
+
}
|
| 104 |
+
} else {
|
| 105 |
+
[[unroll]] for (uint col = tid, idx = 0; idx < num_iters; col += BLOCK_SIZE, ++idx) {
|
| 106 |
+
if (col >= ncols) {
|
| 107 |
+
continue;
|
| 108 |
+
}
|
| 109 |
+
data_d[d_offset + col] = D_TYPE(scale * FLOAT_TYPE(data_a[a_offset + col]));
|
| 110 |
+
}
|
| 111 |
+
}
|
| 112 |
+
#if RMS_NORM_ROPE_FUSION
|
| 113 |
+
barrier();
|
| 114 |
+
rope_params rp = p.rope;
|
| 115 |
+
for (uint t = 2*tid; t < ncols; t += 2*BLOCK_SIZE) {
|
| 116 |
+
if (rp.rope_mode == GGML_ROPE_TYPE_NEOX) {
|
| 117 |
+
rope_neox(t, row, channel, samp, rp);
|
| 118 |
+
} else if (rp.rope_mode == GGML_ROPE_TYPE_NORMAL) {
|
| 119 |
+
rope_norm(t, row, channel, samp, rp);
|
| 120 |
+
}
|
| 121 |
+
}
|
| 122 |
+
#endif
|
| 123 |
+
}
|
| 124 |
+
|
| 125 |
+
void main() {
|
| 126 |
+
// instantiate the rms_norm function for several different
|
| 127 |
+
// dimensions, to allow loop unrolling
|
| 128 |
+
uint num_blocks = (p.ne00 + BLOCK_SIZE - 1) / BLOCK_SIZE;
|
| 129 |
+
if (num_blocks > 32) {
|
| 130 |
+
rms_norm(num_blocks);
|
| 131 |
+
} else if (num_blocks > 16) {
|
| 132 |
+
rms_norm(32);
|
| 133 |
+
} else if (num_blocks > 12) {
|
| 134 |
+
rms_norm(16);
|
| 135 |
+
} else if (num_blocks > 10) {
|
| 136 |
+
rms_norm(12);
|
| 137 |
+
} else if (num_blocks > 8) {
|
| 138 |
+
rms_norm(10);
|
| 139 |
+
} else if (num_blocks > 4) {
|
| 140 |
+
rms_norm(8);
|
| 141 |
+
} else if (num_blocks == 4) {
|
| 142 |
+
rms_norm(4);
|
| 143 |
+
} else if (num_blocks == 3) {
|
| 144 |
+
rms_norm(3);
|
| 145 |
+
} else if (num_blocks == 2) {
|
| 146 |
+
rms_norm(2);
|
| 147 |
+
} else if (num_blocks == 1) {
|
| 148 |
+
rms_norm(1);
|
| 149 |
+
}
|
| 150 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_back.comp
ADDED
|
@@ -0,0 +1,55 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
|
| 3 |
+
#include "generic_head.glsl"
|
| 4 |
+
#include "types.glsl"
|
| 5 |
+
|
| 6 |
+
#extension GL_EXT_control_flow_attributes : enable
|
| 7 |
+
#define BLOCK_SIZE 512
|
| 8 |
+
|
| 9 |
+
layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in;
|
| 10 |
+
|
| 11 |
+
layout (binding = 0) readonly buffer G {A_TYPE data_a[];};
|
| 12 |
+
layout (binding = 1) readonly buffer X {B_TYPE data_b[];};
|
| 13 |
+
layout (binding = 2) writeonly buffer D {D_TYPE data_d[];};
|
| 14 |
+
|
| 15 |
+
shared FLOAT_TYPE sum_xx[BLOCK_SIZE];
|
| 16 |
+
shared FLOAT_TYPE sum_xg[BLOCK_SIZE];
|
| 17 |
+
|
| 18 |
+
void main() {
|
| 19 |
+
const uint row = gl_WorkGroupID.z * 262144 + gl_WorkGroupID.y * 512 + gl_WorkGroupID.x;
|
| 20 |
+
const uint tid = gl_LocalInvocationID.x;
|
| 21 |
+
|
| 22 |
+
// Compute derivative of x[i]/norm(x) = g[i]/norm(x) - x[i] dot(x,g)/KX / norm(x)^1.5
|
| 23 |
+
|
| 24 |
+
// partial sums for thread in warp
|
| 25 |
+
sum_xx[tid] = FLOAT_TYPE(0.0f);
|
| 26 |
+
sum_xg[tid] = FLOAT_TYPE(0.0f);
|
| 27 |
+
|
| 28 |
+
[[unroll]] for (uint col = tid; col < p.KX; col += BLOCK_SIZE) {
|
| 29 |
+
const FLOAT_TYPE gi = FLOAT_TYPE(data_a[row*p.KX + col]);
|
| 30 |
+
const FLOAT_TYPE xi = FLOAT_TYPE(data_b[row*p.KX + col]);
|
| 31 |
+
sum_xx[tid] += xi * xi;
|
| 32 |
+
sum_xg[tid] += xi * gi;
|
| 33 |
+
}
|
| 34 |
+
|
| 35 |
+
// sum up partial sums and write back result
|
| 36 |
+
barrier();
|
| 37 |
+
[[unroll]] for (int s = BLOCK_SIZE / 2; s > 0; s >>= 1) {
|
| 38 |
+
if (tid < s) {
|
| 39 |
+
sum_xx[tid] += sum_xx[tid + s];
|
| 40 |
+
sum_xg[tid] += sum_xg[tid + s];
|
| 41 |
+
}
|
| 42 |
+
barrier();
|
| 43 |
+
}
|
| 44 |
+
|
| 45 |
+
const FLOAT_TYPE eps = FLOAT_TYPE(p.param1);
|
| 46 |
+
const FLOAT_TYPE mean = sum_xx[0] / FLOAT_TYPE(p.KX);
|
| 47 |
+
const FLOAT_TYPE scale_g = inversesqrt(mean + eps);
|
| 48 |
+
const FLOAT_TYPE scale_x = -scale_g * sum_xg[0] / (sum_xx[0] + FLOAT_TYPE(p.KX) * eps);
|
| 49 |
+
|
| 50 |
+
[[unroll]] for (uint col = tid; col < p.KX; col += BLOCK_SIZE) {
|
| 51 |
+
data_d[row*p.KX + col] = D_TYPE(
|
| 52 |
+
scale_g * FLOAT_TYPE(data_a[row*p.KX + col]) +
|
| 53 |
+
scale_x * FLOAT_TYPE(data_b[row*p.KX + col]));
|
| 54 |
+
}
|
| 55 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/rms_norm_partials.comp
ADDED
|
@@ -0,0 +1,65 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
|
| 3 |
+
#include "generic_binary_head.glsl"
|
| 4 |
+
#include "types.glsl"
|
| 5 |
+
|
| 6 |
+
#extension GL_EXT_control_flow_attributes : enable
|
| 7 |
+
#extension GL_KHR_shader_subgroup_arithmetic : enable
|
| 8 |
+
#extension GL_KHR_shader_subgroup_basic : enable
|
| 9 |
+
|
| 10 |
+
#define BLOCK_SIZE 128
|
| 11 |
+
|
| 12 |
+
layout (constant_id = 1) const bool do_multiply = false;
|
| 13 |
+
|
| 14 |
+
layout(local_size_x = BLOCK_SIZE, local_size_y = 1, local_size_z = 1) in;
|
| 15 |
+
|
| 16 |
+
layout (binding = 3, std430) readonly buffer PartialsBuf {float partial_sums[];};
|
| 17 |
+
|
| 18 |
+
shared FLOAT_TYPE sumsh[BLOCK_SIZE];
|
| 19 |
+
|
| 20 |
+
void main() {
|
| 21 |
+
const uint ncols = p.ne00;
|
| 22 |
+
const uint nrows = gl_NumWorkGroups.x;
|
| 23 |
+
const uint nchannels = gl_NumWorkGroups.y;
|
| 24 |
+
|
| 25 |
+
const uint row = 0;
|
| 26 |
+
const uint channel = gl_WorkGroupID.y;
|
| 27 |
+
const uint samp = gl_WorkGroupID.z;
|
| 28 |
+
// The work is split across multiple workgroups in the x dimension. Each invocation
|
| 29 |
+
// processes one element
|
| 30 |
+
const uint tid = gl_GlobalInvocationID.x;
|
| 31 |
+
|
| 32 |
+
const uint stride_row = p.nb01;
|
| 33 |
+
const uint stride_channel = p.nb02;
|
| 34 |
+
const uint stride_sample = p.nb03;
|
| 35 |
+
|
| 36 |
+
uint32_t a_offset = samp*stride_sample + channel*stride_channel + row*stride_row + get_aoffset();
|
| 37 |
+
uint32_t b_offset = src1_idx(0, row, channel, samp) + get_boffset();
|
| 38 |
+
uint32_t d_offset = ((samp*nchannels + channel)*nrows + row)*ncols + get_doffset();
|
| 39 |
+
|
| 40 |
+
FLOAT_TYPE sum = FLOAT_TYPE(0.0f); // partial sum for thread in warp
|
| 41 |
+
|
| 42 |
+
uint32_t num_partials = p.param3;
|
| 43 |
+
for (uint32_t i = gl_SubgroupInvocationID; i < num_partials; i += gl_SubgroupSize) {
|
| 44 |
+
sum += partial_sums[i];
|
| 45 |
+
}
|
| 46 |
+
sum = subgroupAdd(sum);
|
| 47 |
+
|
| 48 |
+
uint col = tid;
|
| 49 |
+
if (col >= ncols) {
|
| 50 |
+
return;
|
| 51 |
+
}
|
| 52 |
+
|
| 53 |
+
const FLOAT_TYPE mean = sum / FLOAT_TYPE(ncols);
|
| 54 |
+
const FLOAT_TYPE scale = inversesqrt(mean + FLOAT_TYPE(p.param1));
|
| 55 |
+
|
| 56 |
+
if (do_multiply) {
|
| 57 |
+
if (ncols > p.ne10) {
|
| 58 |
+
data_d[d_offset + col] = D_TYPE(scale * FLOAT_TYPE(data_a[a_offset + col]) * FLOAT_TYPE(data_b[b_offset + fastmod(col, p.ne10)]));
|
| 59 |
+
} else {
|
| 60 |
+
data_d[d_offset + col] = D_TYPE(scale * FLOAT_TYPE(data_a[a_offset + col]) * FLOAT_TYPE(data_b[b_offset + col]));
|
| 61 |
+
}
|
| 62 |
+
} else {
|
| 63 |
+
data_d[d_offset + col] = D_TYPE(scale * FLOAT_TYPE(data_a[a_offset + col]));
|
| 64 |
+
}
|
| 65 |
+
}
|
llama.cpp/ggml/src/ggml-vulkan/vulkan-shaders/roll.comp
ADDED
|
@@ -0,0 +1,46 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#version 450
|
| 2 |
+
|
| 3 |
+
#include "types.glsl"
|
| 4 |
+
#include "generic_unary_head.glsl"
|
| 5 |
+
|
| 6 |
+
layout(local_size_x = 512, local_size_y = 1, local_size_z = 1) in;
|
| 7 |
+
|
| 8 |
+
uint wrap_idx(int i, uint ne) {
|
| 9 |
+
if (i < 0) {
|
| 10 |
+
return i + ne;
|
| 11 |
+
} else if (i >= ne) {
|
| 12 |
+
return i - ne;
|
| 13 |
+
}
|
| 14 |
+
return i;
|
| 15 |
+
}
|
| 16 |
+
|
| 17 |
+
void main() {
|
| 18 |
+
const uint idx = get_idx();
|
| 19 |
+
if (idx >= p.ne) {
|
| 20 |
+
return;
|
| 21 |
+
}
|
| 22 |
+
|
| 23 |
+
const uint i3 = fastdiv(idx, p.ne1_012mp, fastdiv_L(p.ne1_Ls, 0));
|
| 24 |
+
const uint i3_offset = i3 * p.ne12*p.ne11*p.ne10;
|
| 25 |
+
const uint i2 = fastdiv(idx - i3_offset, p.ne1_01mp, fastdiv_L(p.ne1_Ls, 1));
|
| 26 |
+
const uint i2_offset = i2*p.ne11*p.ne10;
|
| 27 |
+
const uint i1 = fastdiv(idx - i3_offset - i2_offset, p.ne1_0mp, fastdiv_L(p.ne1_Ls, 2));
|
| 28 |
+
const uint i0 = idx - i3_offset - i2_offset - i1*p.ne10;
|
| 29 |
+
|
| 30 |
+
const uint p1 = floatBitsToUint(p.param1);
|
| 31 |
+
const uint p2 = floatBitsToUint(p.param2);
|
| 32 |
+
const int s0 = int(p1 >> 16) - 0x8000;
|
| 33 |
+
const int s1 = int(p1 & 0xFFFF) - 0x8000;
|
| 34 |
+
const int s2 = int(p2 >> 16) - 0x8000;
|
| 35 |
+
const int s3 = int(p2 & 0xFFFF) - 0x8000;
|
| 36 |
+
|
| 37 |
+
const uint i00 = wrap_idx(int(i0) - s0, p.ne10);
|
| 38 |
+
const uint i01 = wrap_idx(int(i1) - s1, p.ne11);
|
| 39 |
+
const uint i02 = wrap_idx(int(i2) - s2, p.ne12);
|
| 40 |
+
const uint i03 = wrap_idx(int(i3) - s3, p.ne13);
|
| 41 |
+
|
| 42 |
+
const uint a_idx = i03*p.nb03 + i02*p.nb02 + i01*p.nb01 + i00*p.nb00;
|
| 43 |
+
const uint d_idx = i3 *p.nb13 + i2 *p.nb12 + i1 *p.nb11 + i0 *p.nb10;
|
| 44 |
+
|
| 45 |
+
data_d[get_doffset() + d_idx] = D_TYPE(data_a[get_aoffset() + a_idx]);
|
| 46 |
+
}
|