kat0012 commited on
Commit
a3a236e
·
verified ·
1 Parent(s): 0f255ff

Add build-info files, CI workflows, build scripts and chat templates missed by staging glob

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .github/workflows/build-3rd-party.yml +57 -0
  2. .github/workflows/build-and-test-snapdragon.yml +148 -0
  3. .github/workflows/build-android.yml +149 -0
  4. .github/workflows/build-apple.yml +287 -0
  5. .github/workflows/build-cache.yml +118 -0
  6. .github/workflows/build-cann.yml +104 -0
  7. .github/workflows/build-cmake-pkg.yml +53 -0
  8. .github/workflows/build-cpu.yml +235 -0
  9. .github/workflows/build-cross.yml +317 -0
  10. .github/workflows/build-cuda-ubuntu.yml +193 -0
  11. .github/workflows/build-cuda-windows.yml +183 -0
  12. .github/workflows/build-ibm.yml +150 -0
  13. .github/workflows/build-msys.yml +70 -0
  14. .github/workflows/build-opencl.yml +92 -0
  15. .github/workflows/build-openvino.yml +178 -0
  16. .github/workflows/build-riscv.yml +188 -0
  17. .github/workflows/build-sanitize.yml +108 -0
  18. .github/workflows/build-self-hosted.yml +409 -0
  19. .github/workflows/build-sycl.yml +171 -0
  20. .github/workflows/build-virtgpu.yml +50 -0
  21. .github/workflows/build-vulkan.yml +232 -0
  22. .github/workflows/build-wasm.yml +110 -0
  23. .github/workflows/build-webgpu.yml +195 -0
  24. cmake/build-info.cmake +48 -0
  25. common/build-info.cpp.in +35 -0
  26. common/build-info.h +13 -0
  27. docs/build-riscv64-spacemit.md +110 -0
  28. docs/build-s390x.md +275 -0
  29. docs/build.md +844 -0
  30. examples/llama.android/app/build.gradle.kts +58 -0
  31. examples/llama.android/build.gradle.kts +6 -0
  32. examples/llama.android/lib/build.gradle.kts +79 -0
  33. examples/sycl/build.sh +53 -0
  34. examples/test-cmake/build-install.sh +19 -0
  35. examples/test-cmake/build.sh +7 -0
  36. models/templates/Apertus-8B-Instruct.jinja +327 -0
  37. models/templates/Apriel-1.6-15b-Thinker-fixed.jinja +171 -0
  38. models/templates/Bielik-11B-v3.0-Instruct.jinja +77 -0
  39. models/templates/ByteDance-Seed-OSS.jinja +171 -0
  40. models/templates/Cohere2MoE.jinja +263 -0
  41. models/templates/CohereForAI-c4ai-command-r-plus-tool_use.jinja +202 -0
  42. models/templates/CohereForAI-c4ai-command-r7b-12-2024-tool_use.jinja +156 -0
  43. models/templates/GLM-4.6.jinja +106 -0
  44. models/templates/GLM-4.7-Flash.jinja +86 -0
  45. models/templates/GigaChat3-10B-A1.8B.jinja +355 -0
  46. models/templates/GigaChat3.1-10B-A1.8B.jinja +339 -0
  47. models/templates/HuggingFaceTB-SmolLM3-3B.jinja +61 -0
  48. models/templates/Kimi-K2-Instruct.jinja +60 -0
  49. models/templates/Kimi-K2-Thinking.jinja +108 -0
  50. models/templates/Kimi-K3.jinja +324 -0
.github/workflows/build-3rd-party.yml ADDED
@@ -0,0 +1,57 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: CI (3rd-party)
2
+
3
+ on:
4
+ workflow_dispatch: # allows manual triggering
5
+ push:
6
+ branches:
7
+ - master
8
+ paths: [
9
+ '.github/workflows/build-3rd-party.yml',
10
+ '**/CMakeLists.txt',
11
+ '**/.cmake',
12
+ '**/*.h',
13
+ '**/*.hpp',
14
+ '**/*.c',
15
+ '**/*.cpp'
16
+ ]
17
+
18
+ concurrency:
19
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
20
+ cancel-in-progress: true
21
+
22
+ env:
23
+ GGML_NLOOP: 3
24
+ GGML_N_THREADS: 1
25
+ LLAMA_ARG_LOG_COLORS: 1
26
+ LLAMA_ARG_LOG_PREFIX: 1
27
+ LLAMA_ARG_LOG_TIMESTAMPS: 1
28
+
29
+ jobs:
30
+ ubuntu-24-llguidance:
31
+ runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
32
+
33
+ steps:
34
+ - name: Clone
35
+ id: checkout
36
+ uses: actions/checkout@v6
37
+
38
+ - name: Dependencies
39
+ id: depends
40
+ run: |
41
+ sudo apt-get update
42
+ sudo apt-get install build-essential libssl-dev
43
+
44
+ - name: Build
45
+ id: cmake_build
46
+ run: |
47
+ cmake -B build \
48
+ -DLLAMA_FATAL_WARNINGS=ON \
49
+ -DLLAMA_LLGUIDANCE=ON
50
+ cmake --build build --config Release -j $(nproc)
51
+
52
+ - name: Test
53
+ id: cmake_test
54
+ run: |
55
+ cd build
56
+ ctest -L main --verbose --timeout 900
57
+
.github/workflows/build-and-test-snapdragon.yml ADDED
@@ -0,0 +1,148 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: CI (snapdragon)
2
+
3
+ on:
4
+ workflow_dispatch:
5
+ push:
6
+ branches:
7
+ - master
8
+ paths:
9
+ - '.github/workflows/build-and-test-snapdragon.yml'
10
+ - 'ggml/include/ggml-hexagon.h'
11
+ - 'ggml/src/ggml-hexagon/**'
12
+ - 'docs/backend/snapdragon/**'
13
+ - 'scripts/snapdragon/**'
14
+ - 'CMakePresets.json'
15
+
16
+ pull_request:
17
+ types: [opened, synchronize, reopened]
18
+ paths:
19
+ - '.github/workflows/build-and-test-snapdragon.yml'
20
+ - 'ggml/include/ggml-hexagon.h'
21
+ - 'ggml/src/ggml-hexagon/**'
22
+ - 'docs/backend/snapdragon/**'
23
+ - 'scripts/snapdragon/**'
24
+ - 'CMakePresets.json'
25
+
26
+ concurrency:
27
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
28
+ cancel-in-progress: true
29
+
30
+ jobs:
31
+ android-ndk-snapdragon:
32
+ runs-on: ubuntu-latest
33
+ container:
34
+ image: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.7'
35
+ defaults:
36
+ run:
37
+ shell: bash
38
+
39
+ steps:
40
+ - name: Clone
41
+ uses: actions/checkout@v6
42
+ with:
43
+ fetch-depth: 0
44
+ lfs: false
45
+
46
+ - name: Build Llama.CPP for Snapdragon Android
47
+ id: build_llama_cpp_snapdragon_android
48
+ run: |
49
+ cp docs/backend/snapdragon/CMakeUserPresets.json .
50
+ cmake --preset arm64-android-snapdragon-release -B build
51
+ cmake --build build
52
+ cmake --install build --prefix pkg-snapdragon/llama.cpp
53
+
54
+ - name: Upload Llama.CPP Snapdragon Android Build Artifact
55
+ if: ${{ always() && steps.build_llama_cpp_snapdragon_android.outcome == 'success' }}
56
+ uses: actions/upload-artifact@v6
57
+ with:
58
+ name: llama-cpp-android-arm64-snapdragon
59
+ path: pkg-snapdragon/llama.cpp
60
+
61
+ linux-iot-snapdragon:
62
+ runs-on: ubuntu-latest
63
+ container:
64
+ image: 'ghcr.io/snapdragon-toolchain/arm64-linux:v0.7'
65
+ defaults:
66
+ run:
67
+ shell: bash
68
+
69
+ steps:
70
+ - name: Clone
71
+ uses: actions/checkout@v6
72
+ with:
73
+ fetch-depth: 0
74
+ lfs: false
75
+
76
+ - name: Build Llama.CPP for Snapdragon Linux IoT
77
+ id: build_llama_cpp_snapdragon_linux
78
+ run: |
79
+ cp docs/backend/snapdragon/CMakeUserPresets.json .
80
+ cmake --preset arm64-linux-snapdragon-release -B build-snapdragon -DGGML_OPENCL=ON
81
+ cmake --build build-snapdragon -j $(nproc)
82
+ cmake --install build-snapdragon --prefix pkg-snapdragon/llama.cpp
83
+
84
+ - name: Upload Llama.CPP Snapdragon Linux IoT Build Artifact
85
+ if: ${{ always() && steps.build_llama_cpp_snapdragon_linux.outcome == 'success' }}
86
+ uses: actions/upload-artifact@v6
87
+ with:
88
+ name: llama-cpp-linux-arm64-snapdragon
89
+ path: pkg-snapdragon/llama.cpp
90
+
91
+ test-snapdragon-qdc:
92
+ name: Test on QDC Device (${{ matrix.device }})
93
+ needs: [android-ndk-snapdragon, linux-iot-snapdragon]
94
+ runs-on: ubuntu-24.04-arm
95
+ timeout-minutes: 90
96
+ strategy:
97
+ fail-fast: false
98
+ matrix:
99
+ device: [SM8750, SM8850, QCS9075M]
100
+
101
+ steps:
102
+ - name: Checkout
103
+ uses: actions/checkout@v6
104
+
105
+ - name: Download build artifact
106
+ uses: actions/download-artifact@v7
107
+ with:
108
+ name: ${{ startsWith(matrix.device, 'QCS') && 'llama-cpp-linux-arm64-snapdragon' || 'llama-cpp-android-arm64-snapdragon' }}
109
+ path: pkg-snapdragon/llama.cpp
110
+
111
+ - name: Set up Python
112
+ uses: actions/setup-python@v6
113
+ with:
114
+ python-version: '3.x'
115
+ cache: pip
116
+
117
+ - name: Install system dependencies
118
+ run: |
119
+ sudo apt-get update
120
+ sudo apt-get install -y curl unzip
121
+
122
+ - name: Install QDC SDK wheel
123
+ run: |
124
+ curl -fSL -o qdc_sdk.zip https://softwarecenter.qualcomm.com/api/download/software/tools/Qualcomm_Device_Cloud_SDK/All/0.2.3/qualcomm_device_cloud_sdk-0.2.3.zip
125
+ unzip qdc_sdk.zip -d qdc_sdk
126
+ pip install qdc_sdk/qualcomm_device_cloud_sdk-0.2.3-py3-none-any.whl
127
+
128
+ - name: Check QDC API key
129
+ id: check_secret
130
+ env:
131
+ QDC_API_KEY: ${{ secrets.QDC_API_KEY }}
132
+ run: echo "has-qdc-key=${{ env.QDC_API_KEY != '' }}" >> "$GITHUB_OUTPUT"
133
+
134
+ - name: Run QDC tests (${{ matrix.device }})
135
+ if: steps.check_secret.outputs.has-qdc-key == 'true'
136
+ run: |
137
+ python scripts/snapdragon/qdc/run_qdc_jobs.py \
138
+ --test all \
139
+ --pkg-dir pkg-snapdragon/llama.cpp \
140
+ --model-url "https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_0.gguf" \
141
+ --device ${{ matrix.device }} \
142
+ ${{ startsWith(matrix.device, 'QCS') && '--retries 2 --retry-delay 300' || '' }}
143
+ env:
144
+ QDC_API_KEY: ${{ secrets.QDC_API_KEY }}
145
+
146
+ - name: Cleanup
147
+ if: always()
148
+ run: rm -rf pkg-snapdragon qdc_sdk qdc_sdk.zip
.github/workflows/build-android.yml ADDED
@@ -0,0 +1,149 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: CI (android)
2
+
3
+ on:
4
+ workflow_dispatch:
5
+ push:
6
+ branches:
7
+ - master
8
+ paths:
9
+ - '.github/workflows/build-android.yml'
10
+ - '**/CMakeLists.txt'
11
+ - '**/.cmake'
12
+ - '**/*.h'
13
+ - '**/*.hpp'
14
+ - '**/*.c'
15
+ - '**/*.cpp'
16
+
17
+ pull_request:
18
+ types: [opened, synchronize, reopened]
19
+ paths:
20
+ - '.github/workflows/build-android.yml'
21
+ - 'examples/llama.android/**'
22
+
23
+ concurrency:
24
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
25
+ cancel-in-progress: true
26
+
27
+ env:
28
+ GGML_NLOOP: 3
29
+ GGML_N_THREADS: 1
30
+ LLAMA_ARG_LOG_COLORS: 1
31
+ LLAMA_ARG_LOG_PREFIX: 1
32
+ LLAMA_ARG_LOG_TIMESTAMPS: 1
33
+
34
+ jobs:
35
+ default:
36
+ runs-on: ubuntu-latest
37
+
38
+ steps:
39
+ - name: Clone
40
+ uses: actions/checkout@v6
41
+ with:
42
+ fetch-depth: 0
43
+ lfs: false
44
+
45
+ - name: Set up JDK
46
+ uses: actions/setup-java@v5
47
+ with:
48
+ java-version: 17
49
+ distribution: zulu
50
+
51
+ - name: Setup Android SDK
52
+ uses: android-actions/setup-android@40fd30fb8d7440372e1316f5d1809ec01dcd3699 # v4.0.1
53
+ with:
54
+ log-accepted-android-sdk-licenses: false
55
+
56
+ - name: Build
57
+ run: |
58
+ cd examples/llama.android
59
+ ./gradlew build --no-daemon
60
+
61
+ ndk:
62
+ runs-on: ubuntu-latest
63
+ container:
64
+ image: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.3'
65
+ defaults:
66
+ run:
67
+ shell: bash
68
+
69
+ steps:
70
+ - name: Clone
71
+ uses: actions/checkout@v6
72
+ with:
73
+ fetch-depth: 0
74
+ lfs: false
75
+
76
+ - name: Dependencies
77
+ run: |
78
+ apt-get update
79
+ apt-get install -y build-essential
80
+
81
+ - name: Build
82
+ id: ndk_build
83
+ run: |
84
+ cmake -D ANDROID_ABI=arm64-v8a -D ANDROID_PLATFORM=android-31 -D CMAKE_TOOLCHAIN_FILE=${ANDROID_NDK_ROOT}/build/cmake/android.toolchain.cmake -D GGML_NATIVE=OFF -DGGML_CPU_ARM_ARCH=armv8.5-a+fp16+i8mm -G Ninja -D LLAMA_OPENSSL=OFF -D GGML_OPENMP=OFF -B build
85
+ cmake --build build
86
+ cmake --install build --prefix pkg-adb/llama.cpp
87
+
88
+ - name: Upload Android Build Artifact
89
+ if: ${{ always() && steps.ndk_build.outcome == 'success' }}
90
+ uses: actions/upload-artifact@v6
91
+ with:
92
+ name: llama-cpp-android-arm64-cpu
93
+ path: pkg-adb/llama.cpp
94
+
95
+ arm64:
96
+ runs-on: ubuntu-latest
97
+
98
+ env:
99
+ NDK_VERSION: "29.0.14206865"
100
+
101
+ steps:
102
+ - name: Clone
103
+ id: checkout
104
+ uses: actions/checkout@v6
105
+
106
+ # note : disabled to spare some cache space (https://github.com/ggml-org/llama.cpp/pull/23789)
107
+ # for some reason, the ccache does not improve the build time in this case
108
+ # example:
109
+ # cache off: https://github.com/ggerganov/tmp2/actions/runs/26534713799/job/78160400831
110
+ # cache on: https://github.com/ggerganov/tmp2/actions/runs/26534713799/job/78224189394
111
+ #
112
+ #- name: ccache
113
+ # uses: ggml-org/ccache-action@v1.2.24
114
+ # with:
115
+ # key: android-ubuntu-arm64
116
+ # evict-old-files: 1d
117
+ # save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
118
+
119
+ - name: Set up JDK
120
+ uses: actions/setup-java@v5
121
+ with:
122
+ java-version: 17
123
+ distribution: temurin
124
+
125
+ - name: Setup Android SDK
126
+ uses: android-actions/setup-android@40fd30fb8d7440372e1316f5d1809ec01dcd3699 # v4.0.1
127
+ with:
128
+ log-accepted-android-sdk-licenses: false
129
+
130
+ - name: Install NDK
131
+ run: |
132
+ sdkmanager "ndk;${{ env.NDK_VERSION }}"
133
+ echo "ANDROID_NDK=${ANDROID_SDK_ROOT}/ndk/${{ env.NDK_VERSION }}" >> $GITHUB_ENV
134
+
135
+ - name: Build
136
+ id: cmake_build
137
+ run: |
138
+ cmake -B build \
139
+ -DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake \
140
+ -DANDROID_ABI=arm64-v8a \
141
+ -DANDROID_PLATFORM=android-28 \
142
+ -DLLAMA_FATAL_WARNINGS=ON \
143
+ -DGGML_BACKEND_DL=ON \
144
+ -DGGML_NATIVE=OFF \
145
+ -DGGML_CPU_ALL_VARIANTS=ON \
146
+ -DGGML_OPENMP=OFF \
147
+ -DLLAMA_BUILD_BORINGSSL=ON \
148
+ -DGGML_RPC=ON
149
+ time cmake --build build --config Release -j $(nproc)
.github/workflows/build-apple.yml ADDED
@@ -0,0 +1,287 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: CI (apple)
2
+
3
+ on:
4
+ workflow_dispatch: # allows manual triggering
5
+ push:
6
+ branches:
7
+ - master
8
+ paths: [
9
+ '.github/workflows/build-apple.yml',
10
+ '**/CMakeLists.txt',
11
+ '**/.cmake',
12
+ '**/*.h',
13
+ '**/*.hpp',
14
+ '**/*.c',
15
+ '**/*.cpp',
16
+ '**/*.swift',
17
+ '**/*.m',
18
+ '**/*.metal'
19
+ ]
20
+
21
+ pull_request:
22
+ types: [opened, synchronize, reopened]
23
+ paths: [
24
+ '.github/workflows/build-apple.yml',
25
+ 'ggml/src/ggml-metal/**',
26
+ 'ggml/src/ggml-rpc/**'
27
+ ]
28
+
29
+ concurrency:
30
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
31
+ cancel-in-progress: true
32
+
33
+ env:
34
+ GGML_NLOOP: 3
35
+ GGML_N_THREADS: 1
36
+ LLAMA_ARG_LOG_COLORS: 1
37
+ LLAMA_ARG_LOG_PREFIX: 1
38
+ LLAMA_ARG_LOG_TIMESTAMPS: 1
39
+
40
+ jobs:
41
+ macos-latest-arm64:
42
+ runs-on: macos-latest
43
+
44
+ steps:
45
+ - name: Clone
46
+ id: checkout
47
+ uses: actions/checkout@v6
48
+
49
+ - name: ccache
50
+ uses: ggml-org/ccache-action@v1.2.24
51
+ with:
52
+ key: apple-arm64
53
+ save: false
54
+
55
+ - name: ccache-buckets-restore
56
+ uses: ./.github/actions/ccache-buckets
57
+ env:
58
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
59
+ with:
60
+ key: apple-arm64
61
+ folder: llama.cpp
62
+ hf_bucket: ggml-org/cache
63
+
64
+ - name: Build
65
+ id: cmake_build
66
+ run: |
67
+ sysctl -a
68
+ cmake -B build \
69
+ -DCMAKE_BUILD_RPATH="@loader_path" \
70
+ -DLLAMA_FATAL_WARNINGS=ON \
71
+ -DLLAMA_BUILD_BORINGSSL=ON \
72
+ -DGGML_METAL_EMBED_LIBRARY=OFF \
73
+ -DGGML_METAL_SHADER_DEBUG=ON \
74
+ -DGGML_RPC=ON \
75
+ -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3
76
+ time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
77
+
78
+ - name: ccache-buckets-save
79
+ if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
80
+ uses: ./.github/actions/ccache-buckets
81
+ env:
82
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
83
+ with:
84
+ key: apple-arm64
85
+ folder: llama.cpp
86
+ evict-old-files: 1d
87
+ hf_bucket: ggml-org/cache
88
+ save: true
89
+
90
+ - name: Check for leaks
91
+ run: |
92
+ cmd=(./build/bin/test-thread-safety -hf ggml-org/gemma-3-270m-qat-GGUF -ngl 99 -p "$(printf 'hello %.0s' {1..128})" -n 16 -c 512 -ub 32 -np 2 -t 2 -lv 1)
93
+ leaks -atExit -- "${cmd[@]}"
94
+ # Graphics devices are leaked by Metal in Apple code sometimes, so we ignore those leaks
95
+ OBJC_DEBUG_MISSING_POOLS=YES "${cmd[@]}" 2>&1 | awk '{ print } index($0, "autoreleased with no pool in place") && !/class [a-zA-Z0-9]+Device autoreleased/ { found = 1 } END { exit found }'
96
+
97
+ - name: Test
98
+ id: cmake_test
99
+ run: |
100
+ cd build
101
+ ctest -L main -E "test-llama-archs" --verbose --timeout 900
102
+
103
+ macos-latest-x64:
104
+ runs-on: macos-15-intel
105
+
106
+ steps:
107
+ - name: Clone
108
+ id: checkout
109
+ uses: actions/checkout@v6
110
+
111
+ - name: ccache
112
+ uses: ggml-org/ccache-action@v1.2.24
113
+ with:
114
+ key: apple-x64
115
+ save: false
116
+
117
+ - name: ccache-buckets-restore
118
+ uses: ./.github/actions/ccache-buckets
119
+ env:
120
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
121
+ with:
122
+ key: apple-x64
123
+ folder: llama.cpp
124
+ hf_bucket: ggml-org/cache
125
+
126
+ - name: Build
127
+ id: cmake_build
128
+ run: |
129
+ sysctl -a
130
+ # Metal is disabled due to intermittent failures with Github runners not having a GPU:
131
+ # https://github.com/ggml-org/llama.cpp/actions/runs/8635935781/job/23674807267#step:5:2313
132
+ cmake -B build \
133
+ -DCMAKE_BUILD_RPATH="@loader_path" \
134
+ -DLLAMA_FATAL_WARNINGS=ON \
135
+ -DLLAMA_BUILD_BORINGSSL=ON \
136
+ -DGGML_METAL=OFF \
137
+ -DGGML_RPC=ON \
138
+ -DCMAKE_OSX_DEPLOYMENT_TARGET=13.3
139
+ time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
140
+
141
+ - name: ccache-buckets-save
142
+ if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
143
+ uses: ./.github/actions/ccache-buckets
144
+ env:
145
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
146
+ with:
147
+ key: apple-x64
148
+ folder: llama.cpp
149
+ evict-old-files: 1d
150
+ hf_bucket: ggml-org/cache
151
+ save: true
152
+
153
+ - name: Test
154
+ id: cmake_test
155
+ run: |
156
+ cd build
157
+ ctest -L main --verbose --timeout 900
158
+
159
+ macos-latest-ios-xcode:
160
+ runs-on: macos-latest
161
+
162
+ steps:
163
+ - name: Checkout code
164
+ uses: actions/checkout@v6
165
+
166
+ - name: Setup Xcode
167
+ uses: ggml-org/setup-xcode@v1
168
+ with:
169
+ xcode-version: latest-stable
170
+
171
+ - name: Build
172
+ id: cmake_build
173
+ run: |
174
+ sysctl -a
175
+ cmake -B build -G Xcode \
176
+ -DGGML_METAL_EMBED_LIBRARY=ON \
177
+ -DLLAMA_OPENSSL=OFF \
178
+ -DLLAMA_BUILD_APP=OFF \
179
+ -DLLAMA_BUILD_EXAMPLES=OFF \
180
+ -DLLAMA_BUILD_TOOLS=OFF \
181
+ -DLLAMA_BUILD_TESTS=OFF \
182
+ -DLLAMA_BUILD_SERVER=OFF \
183
+ -DCMAKE_SYSTEM_NAME=iOS \
184
+ -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
185
+ -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
186
+ cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
187
+
188
+ - name: xcodebuild for swift package
189
+ id: xcodebuild
190
+ run: |
191
+ ./build-xcframework.sh
192
+
193
+ - name: Upload xcframework artifact
194
+ uses: actions/upload-artifact@v6
195
+ with:
196
+ name: llama-xcframework
197
+ path: build-apple/llama.xcframework/
198
+ retention-days: 1
199
+
200
+ - name: Build Xcode project
201
+ run: |
202
+ xcodebuild -downloadPlatform iOS
203
+ xcodebuild -project examples/llama.swiftui/llama.swiftui.xcodeproj -scheme llama.swiftui -sdk iphoneos CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY= -destination 'generic/platform=iOS' FRAMEWORK_FOLDER_PATH=./build-ios build
204
+
205
+ macos-latest-tvos:
206
+ runs-on: macos-latest
207
+
208
+ steps:
209
+ - name: Clone
210
+ id: checkout
211
+ uses: actions/checkout@v6
212
+
213
+ - name: Build
214
+ id: cmake_build
215
+ run: |
216
+ sysctl -a
217
+ cmake -B build -G Xcode \
218
+ -DGGML_METAL_EMBED_LIBRARY=ON \
219
+ -DLLAMA_BUILD_COMMON=OFF \
220
+ -DLLAMA_BUILD_APP=OFF \
221
+ -DLLAMA_BUILD_EXAMPLES=OFF \
222
+ -DLLAMA_BUILD_TOOLS=OFF \
223
+ -DLLAMA_BUILD_TESTS=OFF \
224
+ -DLLAMA_BUILD_SERVER=OFF \
225
+ -DCMAKE_SYSTEM_NAME=tvOS \
226
+ -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
227
+ -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
228
+ cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
229
+
230
+ macos-latest-visionos:
231
+ runs-on: macos-latest
232
+
233
+ steps:
234
+ - name: Clone
235
+ id: checkout
236
+ uses: actions/checkout@v6
237
+
238
+ - name: Build
239
+ id: cmake_build
240
+ run: |
241
+ sysctl -a
242
+ cmake -B build -G Xcode \
243
+ -DGGML_METAL_EMBED_LIBRARY=ON \
244
+ -DLLAMA_BUILD_COMMON=OFF \
245
+ -DLLAMA_BUILD_APP=OFF \
246
+ -DLLAMA_BUILD_EXAMPLES=OFF \
247
+ -DLLAMA_BUILD_TOOLS=OFF \
248
+ -DLLAMA_BUILD_TESTS=OFF \
249
+ -DLLAMA_BUILD_SERVER=OFF \
250
+ -DCMAKE_SYSTEM_NAME=visionOS \
251
+ -DCMAKE_OSX_DEPLOYMENT_TARGET=1.0 \
252
+ -DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
253
+ cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
254
+
255
+ macos-latest-swift:
256
+ runs-on: macos-latest
257
+ needs: macos-latest-ios-xcode
258
+
259
+ strategy:
260
+ matrix:
261
+ destination: ['generic/platform=macOS', 'generic/platform=iOS', 'generic/platform=tvOS']
262
+
263
+ steps:
264
+ - name: Clone
265
+ id: checkout
266
+ uses: actions/checkout@v6
267
+
268
+ - name: Download xcframework artifact
269
+ uses: actions/download-artifact@v7
270
+ with:
271
+ name: llama-xcframework
272
+ path: build-apple/llama.xcframework/
273
+
274
+ - name: Build llama.cpp with CMake
275
+ id: cmake_build
276
+ run: |
277
+ sysctl -a
278
+ cmake -B build -G Xcode \
279
+ -DGGML_METAL_EMBED_LIBRARY=ON \
280
+ -DLLAMA_OPENSSL=OFF \
281
+ -DLLAMA_BUILD_APP=OFF \
282
+ -DLLAMA_BUILD_EXAMPLES=OFF \
283
+ -DLLAMA_BUILD_TOOLS=OFF \
284
+ -DLLAMA_BUILD_TESTS=OFF \
285
+ -DLLAMA_BUILD_SERVER=OFF \
286
+ -DCMAKE_OSX_ARCHITECTURES="arm64;x86_64"
287
+ cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
.github/workflows/build-cache.yml ADDED
@@ -0,0 +1,118 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: Build Actions Cache
2
+
3
+ on:
4
+ workflow_dispatch: # allows manual triggering
5
+ schedule:
6
+ - cron: '0 * * * *'
7
+
8
+ concurrency:
9
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
10
+ cancel-in-progress: true
11
+
12
+ jobs:
13
+ #ubuntu-24-spacemit-cache:
14
+ # runs-on: ubuntu-24.04
15
+
16
+ # env:
17
+ # # Make sure this is in sync with build-linux-cross.yml
18
+ # SPACEMIT_IME_TOOLCHAIN_VERSION: "1.1.2"
19
+
20
+ # steps:
21
+ # - name: Clone
22
+ # id: checkout
23
+ # uses: actions/checkout@v6
24
+
25
+ # - name: Setup Cache
26
+ # uses: actions/cache@v5
27
+ # id: cache-toolchain
28
+ # with:
29
+ # path: ./spacemit_toolchain
30
+ # key: cache-gha-spacemit-ime-toolchain-v${{ env.SPACEMIT_IME_TOOLCHAIN_VERSION }}-${{ runner.os }}
31
+
32
+ # - name: Setup SpacemiT Toolchain
33
+ # if: steps.cache-toolchain.outputs.cache-hit != 'true'
34
+ # uses: ./.github/actions/linux-setup-spacemit
35
+ # with:
36
+ # path: ./spacemit_toolchain
37
+ # version: ${{ env.SPACEMIT_IME_TOOLCHAIN_VERSION }}
38
+
39
+ ubuntu-24-openvino-cache:
40
+ runs-on: ubuntu-24.04
41
+
42
+ env:
43
+ # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
44
+ OPENVINO_VERSION_MAJOR: "2026.3.1"
45
+ OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
46
+
47
+ steps:
48
+ - name: Clone
49
+ id: checkout
50
+ uses: actions/checkout@v6
51
+
52
+ - name: Setup Cache
53
+ uses: actions/cache@v5
54
+ id: cache-openvino
55
+ with:
56
+ path: ./openvino_toolkit
57
+ key: cache-gha-openvino-toolkit-v${{ env.OPENVINO_VERSION_FULL }}-${{ runner.os }}
58
+
59
+ - name: Setup OpenVINO Toolkit
60
+ if: steps.cache-openvino.outputs.cache-hit != 'true'
61
+ uses: ./.github/actions/linux-setup-openvino
62
+ with:
63
+ path: ./openvino_toolkit
64
+ version_major: ${{ env.OPENVINO_VERSION_MAJOR }}
65
+ version_full: ${{ env.OPENVINO_VERSION_FULL }}
66
+
67
+ windows-2022-openvino-cache:
68
+ runs-on: windows-2022
69
+
70
+ env:
71
+ # Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
72
+ OPENVINO_VERSION_MAJOR: "2026.3.1"
73
+ OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
74
+
75
+ steps:
76
+ - name: Clone
77
+ id: checkout
78
+ uses: actions/checkout@v6
79
+
80
+ - name: Setup Cache
81
+ uses: actions/cache@v5
82
+ id: cache-openvino
83
+ with:
84
+ path: ./openvino_toolkit
85
+ key: cache-gha-openvino-toolkit-v${{ env.OPENVINO_VERSION_FULL }}-${{ runner.os }}
86
+
87
+ - name: Setup OpenVINO Toolkit
88
+ if: steps.cache-openvino.outputs.cache-hit != 'true'
89
+ uses: ./.github/actions/windows-setup-openvino
90
+ with:
91
+ path: ./openvino_toolkit
92
+ version_major: ${{ env.OPENVINO_VERSION_MAJOR }}
93
+ version_full: ${{ env.OPENVINO_VERSION_FULL }}
94
+
95
+ # windows-2022-rocm-cache:
96
+ # runs-on: windows-2022
97
+
98
+ # env:
99
+ # # Make sure this is in sync with release.yml and build-cuda-windows.yml
100
+ # ROCM_VERSION: "7.14.0"
101
+
102
+ # steps:
103
+ # - name: Clone
104
+ # id: checkout
105
+ # uses: actions/checkout@v6
106
+
107
+ # - name: Setup Cache
108
+ # uses: actions/cache@v5
109
+ # id: cache-rocm
110
+ # with:
111
+ # path: C:\TheRock\build
112
+ # key: rocm-wheels-${{ env.ROCM_VERSION }}-multi-arch-${{ runner.os }}
113
+
114
+ # - name: Setup ROCm
115
+ # if: steps.cache-rocm.outputs.cache-hit != 'true'
116
+ # uses: ./.github/actions/windows-setup-rocm
117
+ # with:
118
+ # version: ${{ env.ROCM_VERSION }}
.github/workflows/build-cann.yml ADDED
@@ -0,0 +1,104 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: CI (cann)
2
+
3
+ on:
4
+ workflow_dispatch: # allows manual triggering
5
+ push:
6
+ branches:
7
+ - master
8
+ paths: [
9
+ '.github/workflows/build-cann.yml',
10
+ '**/CMakeLists.txt',
11
+ '**/.cmake',
12
+ '**/*.h',
13
+ '**/*.hpp',
14
+ '**/*.c',
15
+ '**/*.cpp'
16
+ ]
17
+
18
+ pull_request:
19
+ types: [opened, synchronize, reopened]
20
+ paths: [
21
+ '.github/workflows/build-cann.yml',
22
+ 'ggml/src/ggml-cann/**'
23
+ ]
24
+
25
+ concurrency:
26
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
27
+ cancel-in-progress: true
28
+
29
+ env:
30
+ GGML_NLOOP: 3
31
+ GGML_N_THREADS: 1
32
+ LLAMA_ARG_LOG_COLORS: 1
33
+ LLAMA_ARG_LOG_PREFIX: 1
34
+ LLAMA_ARG_LOG_TIMESTAMPS: 1
35
+
36
+ jobs:
37
+ # TODO: this build is disabled to save Github Actions resources (https://github.com/ggml-org/llama.cpp/pull/23705)
38
+ # in order to enable it again, we have to provision dedicated runners to run it
39
+ # openEuler-latest-cann:
40
+ # defaults:
41
+ # run:
42
+ # shell: bash -el {0}
43
+ # strategy:
44
+ # matrix:
45
+ # arch: [x86, aarch64]
46
+ # chip_type: ['910b', '310p']
47
+ # build: ['Release']
48
+ # use_acl_graph: ['on', 'off']
49
+ # exclude:
50
+ # # 310P does not support USE_ACL_GRAPH=on
51
+ # - chip_type: '310p'
52
+ # use_acl_graph: 'on'
53
+ # runs-on: ${{ matrix.arch == 'aarch64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
54
+ # steps:
55
+ # - name: Checkout
56
+ # uses: actions/checkout@v6
57
+ # with:
58
+ # fetch-depth: 0
59
+ #
60
+ # - name: Free up disk space
61
+ # uses: ggml-org/free-disk-space@v1.3.1
62
+ # with:
63
+ # tool-cache: true
64
+ #
65
+ # - name: Set container image
66
+ # id: cann-image
67
+ # run: |
68
+ # image="ascendai/cann:${{ matrix.chip_type == '910b' && '8.5.0-910b-openeuler24.03-py3.11' || '8.5.0-310p-openeuler24.03-py3.11' }}"
69
+ # echo "image=${image}" >> "${GITHUB_OUTPUT}"
70
+ #
71
+ # - name: Pull container image
72
+ # run: docker pull "${{ steps.cann-image.outputs.image }}"
73
+ #
74
+ # - name: Build
75
+ # env:
76
+ # BUILD_TYPE: ${{ matrix.build }}
77
+ # SOC_TYPE: ascend${{ matrix.chip_type }}
78
+ # USE_ACL_GRAPH: ${{ matrix.use_acl_graph }}
79
+ # run: |
80
+ # HOST_UID=$(id -u)
81
+ # HOST_GID=$(id -g)
82
+ #
83
+ # docker run --rm \
84
+ # -v "${PWD}:/workspace" \
85
+ # -w /workspace \
86
+ # -e SOC_TYPE=${SOC_TYPE} \
87
+ # -e BUILD_TYPE=${BUILD_TYPE} \
88
+ # -e USE_ACL_GRAPH=${USE_ACL_GRAPH} \
89
+ # "${{ steps.cann-image.outputs.image }}" \
90
+ # bash -lc '
91
+ # set -e
92
+ # yum install -y --setopt=install_weak_deps=False --setopt=tsflags=nodocs git gcc gcc-c++ make cmake openssl-devel
93
+ # yum clean all && rm -rf /var/cache/yum
94
+ # git config --global --add safe.directory "/workspace"
95
+ # export LD_LIBRARY_PATH=${ASCEND_TOOLKIT_HOME}/lib64:${ASCEND_TOOLKIT_HOME}/$(uname -m)-linux/devlib/:${LD_LIBRARY_PATH}
96
+ # cmake -S . -B build \
97
+ # -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
98
+ # -DGGML_CANN=on \
99
+ # -DSOC_TYPE=${SOC_TYPE} \
100
+ # -DUSE_ACL_GRAPH=${USE_ACL_GRAPH}
101
+ # cmake --build build -j $(nproc)
102
+ #
103
+ # chown -R '"${HOST_UID}"':'"${HOST_GID}"' /workspace/build
104
+ # '
.github/workflows/build-cmake-pkg.yml ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: Build relocatable cmake package
2
+ on:
3
+ workflow_dispatch:
4
+ workflow_call:
5
+
6
+ jobs:
7
+ linux:
8
+ runs-on: [self-hosted, Linux]
9
+ steps:
10
+ - uses: actions/checkout@v6
11
+ with:
12
+ fetch-depth: 0
13
+
14
+ - name: Build
15
+ run: |
16
+ PREFIX="$(pwd)"/inst
17
+ cmake -S . -B build \
18
+ -DCMAKE_PREFIX_PATH="$PREFIX" \
19
+ -DLLAMA_OPENSSL=OFF \
20
+ -DLLAMA_BUILD_TESTS=OFF \
21
+ -DLLAMA_BUILD_TOOLS=OFF \
22
+ -DLLAMA_BUILD_EXAMPLES=OFF \
23
+ -DLLAMA_BUILD_APP=OFF \
24
+ -DLLAMA_BUILD_IS_DEV=OFF \
25
+ -DCMAKE_BUILD_TYPE=Release
26
+ cmake --build build --config Release -j $(nproc)
27
+ cmake --install build --prefix "$PREFIX" --config Release
28
+
29
+ export LLAMA_CONFIG="$PREFIX"/lib/cmake/llama/llama-config.cmake
30
+ build_commit=$(git rev-parse --short HEAD | xargs)
31
+ build_number=$(git rev-list --count HEAD | xargs)
32
+
33
+ major=$(grep -oE "set\(LLAMA_VERSION_MAJOR[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
34
+ minor=$(grep -oE "set\(LLAMA_VERSION_MINOR[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
35
+ patch=$(grep -oE "set\(LLAMA_VERSION_PATCH[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
36
+ build_version="$major.$minor.$patch"
37
+
38
+ checks=("set\(LLAMA_VERSION[[:space:]]+$build_version\)"
39
+ "set\(LLAMA_BUILD_COMMIT[[:space:]]+$build_commit\)"
40
+ "set\(LLAMA_BUILD_NUMBER[[:space:]]+$build_number\)")
41
+
42
+ for check in "${checks[@]}"; do
43
+ if ! grep -qE "$check" "$LLAMA_CONFIG"; then
44
+ echo "Checking llama-config.cmake version... \"$check\" failed!"
45
+ exit 1
46
+ fi
47
+ done
48
+
49
+ echo "Checking llama-config.cmake version... success."
50
+
51
+ cd examples/simple-cmake-pkg
52
+ cmake -S . -B build -DCMAKE_PREFIX_PATH="$PREFIX"/lib/cmake
53
+ cmake --build build -j $(nproc)
.github/workflows/build-cpu.yml ADDED
@@ -0,0 +1,235 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: CI (cpu)
2
+
3
+ on:
4
+ workflow_dispatch: # allows manual triggering
5
+ push:
6
+ branches:
7
+ - master
8
+ paths: [
9
+ '.github/workflows/build-cpu.yml',
10
+ '.github/workflows/build-cmake-pkg.yml',
11
+ '**/CMakeLists.txt',
12
+ '**/.cmake',
13
+ '**/*.h',
14
+ '**/*.hpp',
15
+ '**/*.c',
16
+ '**/*.cpp',
17
+ ]
18
+
19
+ pull_request:
20
+ types: [opened, synchronize, reopened]
21
+ paths: [
22
+ '.github/workflows/build-cpu.yml',
23
+ '.github/workflows/build-cmake-pkg.yml',
24
+ 'ggml/src/ggml-rpc/**',
25
+ '**/CMakeLists.txt',
26
+ '**/.cmake',
27
+ '**/*.h',
28
+ '**/*.hpp',
29
+ '**/*.c',
30
+ '**/*.cpp'
31
+ ]
32
+
33
+ concurrency:
34
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
35
+ cancel-in-progress: true
36
+
37
+ env:
38
+ GGML_NLOOP: 3
39
+ GGML_N_THREADS: 1
40
+ LLAMA_ARG_LOG_COLORS: 1
41
+ LLAMA_ARG_LOG_PREFIX: 1
42
+ LLAMA_ARG_LOG_TIMESTAMPS: 1
43
+
44
+ jobs:
45
+ build-cmake-pkg:
46
+ uses: ./.github/workflows/build-cmake-pkg.yml
47
+
48
+ ubuntu:
49
+ strategy:
50
+ matrix:
51
+ include:
52
+ - build: 'x64'
53
+ os: ubuntu-22.04
54
+ - build: 'arm64'
55
+ os: ubuntu-24.04-arm
56
+
57
+ runs-on: ${{ matrix.os }}
58
+
59
+ steps:
60
+ - name: Clone
61
+ id: checkout
62
+ uses: actions/checkout@v6
63
+
64
+ - name: ccache
65
+ uses: ggml-org/ccache-action@v1.2.24
66
+ with:
67
+ key: cpu-${{ matrix.os }}
68
+ save: false
69
+
70
+ - name: Build Dependencies
71
+ id: build_depends
72
+ run: |
73
+ sudo apt-get update
74
+ sudo apt-get install -y --no-install-recommends \
75
+ python3 python3-pip python3-dev python3-wheel \
76
+ libjpeg-dev build-essential libssl-dev \
77
+ git-lfs
78
+
79
+ - name: Toolchain workaround (GCC 14)
80
+ if: ${{ contains(matrix.os, 'ubuntu-24.04') }}
81
+ run: |
82
+ sudo apt-get install -y gcc-14 g++-14
83
+ echo "CC=gcc-14" >> "$GITHUB_ENV"
84
+ echo "CXX=g++-14" >> "$GITHUB_ENV"
85
+
86
+ - name: Python Dependencies
87
+ id: python_depends
88
+ run: |
89
+ export PIP_BREAK_SYSTEM_PACKAGES="1"
90
+ python3 -m pip install --upgrade pip setuptools
91
+ pip3 install ./gguf-py
92
+
93
+ - name: ccache-buckets-restore
94
+ uses: ./.github/actions/ccache-buckets
95
+ env:
96
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
97
+ with:
98
+ key: cpu-${{ matrix.os }}
99
+ folder: llama.cpp
100
+ hf_bucket: ggml-org/cache
101
+
102
+ - name: Build
103
+ id: cmake_build
104
+ run: |
105
+ cmake -B build \
106
+ -DGGML_NATIVE=OFF \
107
+ -DLLAMA_FATAL_WARNINGS=ON \
108
+ -DGGML_RPC=ON
109
+ time cmake --build build --config Release -j $(nproc)
110
+
111
+ - name: ccache-buckets-save
112
+ if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
113
+ uses: ./.github/actions/ccache-buckets
114
+ env:
115
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
116
+ with:
117
+ key: cpu-${{ matrix.os }}
118
+ folder: llama.cpp
119
+ evict-old-files: 1d
120
+ hf_bucket: ggml-org/cache
121
+ save: true
122
+
123
+ - name: Test
124
+ id: cmake_test
125
+ run: |
126
+ cd build
127
+ ctest -L main --verbose --timeout 900
128
+
129
+ - name: Test llama2c conversion
130
+ id: llama2c_test
131
+ run: |
132
+ cd build
133
+ echo "Fetch tokenizer"
134
+ wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/tok512.bin
135
+ echo "Fetch llama2c model"
136
+ wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/stories260K.bin
137
+ ./bin/llama-convert-llama2c-to-ggml --copy-vocab-from-model ./tok512.bin --llama2c-model stories260K.bin --llama2c-output-model stories260K.gguf
138
+ ./bin/llama-completion -m stories260K.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256
139
+
140
+ windows:
141
+ name: windows / ${{ matrix.build }}
142
+ runs-on: windows-2025
143
+
144
+ env:
145
+ OPENBLAS_VERSION: 0.3.23
146
+ SDE_VERSION: 9.33.0-2024-01-07
147
+
148
+ strategy:
149
+ matrix:
150
+ include:
151
+ - build: 'x64-cpu-static'
152
+ arch: 'x64'
153
+ defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DGGML_NATIVE=OFF -DGGML_OPENMP_FETCH=ON -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=OFF'
154
+ - build: 'x64-openblas'
155
+ arch: 'x64'
156
+ defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_OPENMP=OFF -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/openblas.lib"'
157
+ - build: 'arm64'
158
+ arch: 'arm64'
159
+ defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DGGML_NATIVE=OFF -DGGML_OPENMP_FETCH=ON -DLLAMA_BUILD_SERVER=ON'
160
+
161
+ steps:
162
+ - name: Clone
163
+ id: checkout
164
+ uses: actions/checkout@v6
165
+
166
+ - name: ccache
167
+ uses: ggml-org/ccache-action@v1.2.24
168
+ with:
169
+ key: cpu-windows-2025-${{ matrix.build }}
170
+ variant: ccache
171
+ evict-old-files: 1d
172
+ save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
173
+
174
+ - name: Download OpenBLAS
175
+ id: get_openblas
176
+ if: ${{ matrix.build == 'x64-openblas' }}
177
+ run: |
178
+ curl.exe -o $env:RUNNER_TEMP/openblas.zip -L "https://github.com/xianyi/OpenBLAS/releases/download/v${env:OPENBLAS_VERSION}/OpenBLAS-${env:OPENBLAS_VERSION}-x64.zip"
179
+ curl.exe -o $env:RUNNER_TEMP/OpenBLAS.LICENSE.txt -L "https://github.com/xianyi/OpenBLAS/raw/v${env:OPENBLAS_VERSION}/LICENSE"
180
+ mkdir $env:RUNNER_TEMP/openblas
181
+ tar.exe -xvf $env:RUNNER_TEMP/openblas.zip -C $env:RUNNER_TEMP/openblas
182
+ $vcdir = $(vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath)
183
+ $msvc = $(join-path $vcdir $('VC\Tools\MSVC\'+$(gc -raw $(join-path $vcdir 'VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt')).Trim()))
184
+ $lib = $(join-path $msvc 'bin\Hostx64\x64\lib.exe')
185
+ & $lib /machine:x64 "/def:${env:RUNNER_TEMP}/openblas/lib/libopenblas.def" "/out:${env:RUNNER_TEMP}/openblas/lib/openblas.lib" /name:openblas.dll
186
+
187
+ - name: Install Ninja
188
+ id: install_ninja
189
+ run: |
190
+ choco install ninja
191
+
192
+ - name: Build
193
+ id: cmake_build
194
+ run: |
195
+ cmake -S . -B build ${{ matrix.defines }} `
196
+ -DLLAMA_BUILD_BORINGSSL=ON
197
+ cmake --build build --config Release -j ${env:NUMBER_OF_PROCESSORS}
198
+
199
+ - name: Add libopenblas.dll
200
+ id: add_libopenblas_dll
201
+ if: ${{ matrix.build == 'x64-openblas' }}
202
+ run: |
203
+ cp $env:RUNNER_TEMP/openblas/bin/libopenblas.dll ./build/bin/Release/openblas.dll
204
+ cp $env:RUNNER_TEMP/OpenBLAS.LICENSE.txt ./build/bin/Release/OpenBLAS-${env:OPENBLAS_VERSION}.txt
205
+
206
+ - name: Test
207
+ id: cmake_test
208
+ if: ${{ matrix.arch == 'x64' }}
209
+ run: |
210
+ cd build
211
+ ctest -L main -C Release --verbose --timeout 900
212
+
213
+ # TODO: disabled for now, consider adding tests for all CPU variants instead
214
+ # - name: Test (Intel SDE)
215
+ # id: cmake_test_sde
216
+ # if: ${{ matrix.build == 'avx512-x64' && env.HAS_AVX512F == '0' }} # use Intel SDE for AVX-512 emulation
217
+ # run: |
218
+ # curl.exe -o $env:RUNNER_TEMP/sde.tar.xz -L "https://downloadmirror.intel.com/813591/sde-external-${env:SDE_VERSION}-win.tar.xz"
219
+ # # for some weird reason windows tar doesn't like sde tar.xz
220
+ # 7z x "-o${env:RUNNER_TEMP}" $env:RUNNER_TEMP/sde.tar.xz
221
+ # 7z x "-o${env:RUNNER_TEMP}" $env:RUNNER_TEMP/sde.tar
222
+ # $sde = $(join-path $env:RUNNER_TEMP sde-external-${env:SDE_VERSION}-win/sde.exe)
223
+ # cd build
224
+ # $env:LLAMA_SKIP_TESTS_SLOW_ON_EMULATOR = 1
225
+ # & $sde -future -- ctest -L main -C Release --verbose --timeout 900
226
+
227
+ - name: ccache-clear
228
+ uses: ./.github/actions/ccache-clear
229
+ env:
230
+ GH_TOKEN: ${{ github.token }}
231
+ with:
232
+ key: cpu-windows-2025-${{ matrix.build }}
233
+ older: 5m
234
+ min: 1
235
+ dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
.github/workflows/build-cross.yml ADDED
@@ -0,0 +1,317 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: CI (cross)
2
+ on:
3
+ # only manual triggers due to low-importance of the workflows
4
+ # TODO: for regular runs, provision dedicated self-hosted runners
5
+ workflow_dispatch:
6
+ push:
7
+ branches:
8
+ - master
9
+ paths: [
10
+ '.github/workflows/build-cross.yml',
11
+ 'ggml/src/spacemit/*',
12
+ 'ggml/src/arch/loongarch/*'
13
+ ]
14
+ # run once every week
15
+ schedule:
16
+ - cron: '0 0 * * 0'
17
+
18
+ concurrency:
19
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
20
+ cancel-in-progress: true
21
+
22
+
23
+ jobs:
24
+ # ubuntu-24-riscv64-cpu-cross:
25
+ # runs-on: ubuntu-24.04
26
+
27
+ # steps:
28
+ # - uses: actions/checkout@v6
29
+ # - name: Setup Riscv
30
+ # run: |
31
+ # sudo dpkg --add-architecture riscv64
32
+
33
+ # # Add arch-specific repositories for non-amd64 architectures
34
+ # cat << EOF | sudo tee /etc/apt/sources.list.d/riscv64-ports.list
35
+ # deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ noble main universe
36
+ # deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ noble-updates main universe
37
+ # deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ noble-security main universe
38
+ # deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ noble-backports main universe
39
+ # EOF
40
+
41
+ # sudo apt-get update || true ;# Prevent failure due to missing URLs.
42
+
43
+ # sudo apt-get install -y --no-install-recommends \
44
+ # build-essential \
45
+ # gcc-14-riscv64-linux-gnu \
46
+ # g++-14-riscv64-linux-gnu
47
+
48
+ # - name: Build
49
+ # run: |
50
+ # cmake -B build -DLLAMA_OPENSSL=OFF \
51
+ # -DCMAKE_BUILD_TYPE=Release \
52
+ # -DGGML_OPENMP=OFF \
53
+ # -DLLAMA_BUILD_EXAMPLES=ON \
54
+ # -DLLAMA_BUILD_TOOLS=ON \
55
+ # -DLLAMA_BUILD_TESTS=OFF \
56
+ # -DCMAKE_SYSTEM_NAME=Linux \
57
+ # -DCMAKE_SYSTEM_PROCESSOR=riscv64 \
58
+ # -DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \
59
+ # -DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14 \
60
+ # -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
61
+ # -DCMAKE_FIND_ROOT_PATH=/usr/lib/riscv64-linux-gnu \
62
+ # -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \
63
+ # -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY \
64
+ # -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH
65
+
66
+ # cmake --build build --config Release -j $(nproc)
67
+
68
+ # ubuntu-24-riscv64-vulkan-cross:
69
+ # runs-on: ubuntu-24.04
70
+
71
+ # steps:
72
+ # - uses: actions/checkout@v6
73
+ # - name: Setup Riscv
74
+ # run: |
75
+ # sudo dpkg --add-architecture riscv64
76
+
77
+ # # Add arch-specific repositories for non-amd64 architectures
78
+ # cat << EOF | sudo tee /etc/apt/sources.list.d/riscv64-ports.list
79
+ # deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ noble main universe
80
+ # deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ noble-updates main universe
81
+ # deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ noble-security main universe
82
+ # deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ noble-backports main universe
83
+ # EOF
84
+
85
+ # sudo apt-get update || true ;# Prevent failure due to missing URLs.
86
+
87
+ # sudo apt-get install -y --no-install-recommends \
88
+ # build-essential \
89
+ # glslc \
90
+ # gcc-14-riscv64-linux-gnu \
91
+ # g++-14-riscv64-linux-gnu \
92
+ # libvulkan-dev:riscv64
93
+
94
+ # - name: Build
95
+ # run: |
96
+ # cmake -B build -DLLAMA_OPENSSL=OFF \
97
+ # -DCMAKE_BUILD_TYPE=Release \
98
+ # -DGGML_VULKAN=ON \
99
+ # -DGGML_OPENMP=OFF \
100
+ # -DLLAMA_BUILD_EXAMPLES=ON \
101
+ # -DLLAMA_BUILD_TOOLS=ON \
102
+ # -DLLAMA_BUILD_TESTS=OFF \
103
+ # -DCMAKE_SYSTEM_NAME=Linux \
104
+ # -DCMAKE_SYSTEM_PROCESSOR=riscv64 \
105
+ # -DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \
106
+ # -DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14 \
107
+ # -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
108
+ # -DCMAKE_FIND_ROOT_PATH=/usr/lib/riscv64-linux-gnu \
109
+ # -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \
110
+ # -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY \
111
+ # -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH
112
+
113
+ # cmake --build build --config Release -j $(nproc)
114
+
115
+ # ubuntu-24-arm64-vulkan-cross:
116
+ # runs-on: ubuntu-24.04
117
+
118
+ # steps:
119
+ # - uses: actions/checkout@v6
120
+ # - name: Setup Arm64
121
+ # run: |
122
+ # sudo dpkg --add-architecture arm64
123
+
124
+ # # Add arch-specific repositories for non-amd64 architectures
125
+ # cat << EOF | sudo tee /etc/apt/sources.list.d/arm64-ports.list
126
+ # deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ noble main universe
127
+ # deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ noble-updates main universe
128
+ # deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ noble-security main universe
129
+ # deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ noble-backports main universe
130
+ # EOF
131
+
132
+ # sudo apt-get update || true ;# Prevent failure due to missing URLs.
133
+
134
+ # sudo apt-get install -y --no-install-recommends \
135
+ # build-essential \
136
+ # glslc \
137
+ # crossbuild-essential-arm64 \
138
+ # libvulkan-dev:arm64
139
+
140
+ # - name: Build
141
+ # run: |
142
+ # cmake -B build -DLLAMA_OPENSSL=OFF \
143
+ # -DCMAKE_BUILD_TYPE=Release \
144
+ # -DGGML_VULKAN=ON \
145
+ # -DGGML_OPENMP=OFF \
146
+ # -DLLAMA_BUILD_EXAMPLES=ON \
147
+ # -DLLAMA_BUILD_TOOLS=ON \
148
+ # -DLLAMA_BUILD_TESTS=OFF \
149
+ # -DCMAKE_SYSTEM_NAME=Linux \
150
+ # -DCMAKE_SYSTEM_PROCESSOR=aarch64 \
151
+ # -DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc \
152
+ # -DCMAKE_CXX_COMPILER=aarch64-linux-gnu-g++ \
153
+ # -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
154
+ # -DCMAKE_FIND_ROOT_PATH=/usr/lib/aarch64-linux-gnu \
155
+ # -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \
156
+ # -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY \
157
+ # -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH
158
+
159
+ # cmake --build build --config Release -j $(nproc)
160
+
161
+ debian-13-loongarch64-cpu-cross:
162
+ runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
163
+ container: debian@sha256:653dfb9f86c3782e8369d5f7d29bb8faba1f4bff9025db46e807fa4c22903671
164
+
165
+ steps:
166
+ - uses: actions/checkout@v6
167
+ - name: Setup LoongArch
168
+ run: |
169
+ rm -f /etc/apt/sources.list.d/*
170
+ cat << EOF | tee /etc/apt/sources.list.d/debian-ports.list
171
+ deb http://snapshot.debian.org/archive/debian/20250515T202920Z/ trixie main
172
+ EOF
173
+ ( echo 'quiet "true";'; \
174
+ echo 'APT::Get::Assume-Yes "true";'; \
175
+ echo 'APT::Install-Recommends "false";'; \
176
+ echo 'Acquire::Check-Valid-Until "false";'; \
177
+ echo 'Acquire::Retries "5";'; \
178
+ ) > /etc/apt/apt.conf.d/99snapshot-repos
179
+
180
+ apt-get update
181
+ apt-get install -y ca-certificates debian-ports-archive-keyring cmake git zip
182
+ dpkg --add-architecture loong64
183
+
184
+ # Add arch-specific repositories for non-amd64 architectures
185
+ cat << EOF | tee /etc/apt/sources.list.d/loong64-ports.list
186
+ deb [arch=loong64] http://snapshot.debian.org/archive/debian-ports/20250515T194251Z/ sid main
187
+ EOF
188
+
189
+ apt-get update || true ;# Prevent failure due to missing URLs.
190
+
191
+ apt-get install -y --no-install-recommends \
192
+ build-essential \
193
+ gcc-14-loongarch64-linux-gnu \
194
+ g++-14-loongarch64-linux-gnu
195
+
196
+ - name: Build
197
+ run: |
198
+ cmake -B build -DLLAMA_OPENSSL=OFF \
199
+ -DCMAKE_BUILD_TYPE=Release \
200
+ -DGGML_OPENMP=OFF \
201
+ -DLLAMA_BUILD_EXAMPLES=ON \
202
+ -DLLAMA_BUILD_TOOLS=ON \
203
+ -DLLAMA_BUILD_TESTS=OFF \
204
+ -DCMAKE_SYSTEM_NAME=Linux \
205
+ -DCMAKE_SYSTEM_PROCESSOR=loongarch64 \
206
+ -DCMAKE_C_COMPILER=loongarch64-linux-gnu-gcc-14 \
207
+ -DCMAKE_CXX_COMPILER=loongarch64-linux-gnu-g++-14 \
208
+ -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
209
+ -DCMAKE_FIND_ROOT_PATH=/usr/lib/loongarch64-linux-gnu \
210
+ -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \
211
+ -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY \
212
+ -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH
213
+
214
+ cmake --build build --config Release -j $(nproc)
215
+
216
+ debian-13-loongarch64-vulkan-cross:
217
+ runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
218
+ container: debian@sha256:653dfb9f86c3782e8369d5f7d29bb8faba1f4bff9025db46e807fa4c22903671
219
+
220
+ steps:
221
+ - uses: actions/checkout@v6
222
+ - name: Setup LoongArch
223
+ run: |
224
+ rm -f /etc/apt/sources.list.d/*
225
+ cat << EOF | tee /etc/apt/sources.list.d/debian-ports.list
226
+ deb http://snapshot.debian.org/archive/debian/20250515T202920Z/ trixie main
227
+ EOF
228
+ ( echo 'quiet "true";'; \
229
+ echo 'APT::Get::Assume-Yes "true";'; \
230
+ echo 'APT::Install-Recommends "false";'; \
231
+ echo 'Acquire::Check-Valid-Until "false";'; \
232
+ echo 'Acquire::Retries "5";'; \
233
+ ) > /etc/apt/apt.conf.d/99snapshot-repos
234
+
235
+ apt-get update
236
+ apt-get install -y ca-certificates debian-ports-archive-keyring cmake git zip
237
+ dpkg --add-architecture loong64
238
+
239
+ # Add arch-specific repositories for non-amd64 architectures
240
+ cat << EOF | tee /etc/apt/sources.list.d/loong64-ports.list
241
+ deb [arch=loong64] http://snapshot.debian.org/archive/debian-ports/20250515T194251Z/ sid main
242
+ EOF
243
+
244
+ apt-get update || true ;# Prevent failure due to missing URLs.
245
+
246
+ apt-get install -y --no-install-recommends \
247
+ build-essential \
248
+ glslc \
249
+ spirv-headers \
250
+ gcc-14-loongarch64-linux-gnu \
251
+ g++-14-loongarch64-linux-gnu \
252
+ libvulkan-dev:loong64
253
+
254
+ - name: Build
255
+ run: |
256
+ cmake -B build -DLLAMA_OPENSSL=OFF \
257
+ -DCMAKE_BUILD_TYPE=Release \
258
+ -DGGML_VULKAN=ON \
259
+ -DGGML_OPENMP=OFF \
260
+ -DLLAMA_BUILD_EXAMPLES=ON \
261
+ -DLLAMA_BUILD_TOOLS=ON \
262
+ -DLLAMA_BUILD_TESTS=OFF \
263
+ -DCMAKE_SYSTEM_NAME=Linux \
264
+ -DCMAKE_SYSTEM_PROCESSOR=loongarch64 \
265
+ -DCMAKE_C_COMPILER=loongarch64-linux-gnu-gcc-14 \
266
+ -DCMAKE_CXX_COMPILER=loongarch64-linux-gnu-g++-14 \
267
+ -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
268
+ -DCMAKE_FIND_ROOT_PATH=/usr/lib/loongarch64-linux-gnu \
269
+ -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \
270
+ -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY \
271
+ -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH
272
+
273
+ cmake --build build --config Release -j $(nproc)
274
+
275
+ ubuntu-24-riscv64-cpu-spacemit-ime-cross:
276
+ runs-on: ubuntu-24.04
277
+
278
+ env:
279
+ # Make sure this is in sync with build-cache.yml
280
+ SPACEMIT_IME_TOOLCHAIN_VERSION: "1.2.4"
281
+
282
+ steps:
283
+ - uses: actions/checkout@v6
284
+
285
+ #- name: Use SpacemiT Toolchain Cache
286
+ # uses: actions/cache@v5
287
+ # id: cache-toolchain
288
+ # with:
289
+ # path: ./spacemit_toolchain
290
+ # key: cache-gha-spacemit-ime-toolchain-v${{ env.SPACEMIT_IME_TOOLCHAIN_VERSION }}-${{ runner.os }}
291
+
292
+ - name: Setup SpacemiT Toolchain
293
+ #if: steps.cache-toolchain.outputs.cache-hit != 'true'
294
+ uses: ./.github/actions/linux-setup-spacemit
295
+ with:
296
+ path: ./spacemit_toolchain
297
+ version: ${{ env.SPACEMIT_IME_TOOLCHAIN_VERSION }}
298
+
299
+ - name: Build
300
+ run: |
301
+ export RISCV_ROOT_PATH=${PWD}/spacemit_toolchain
302
+ cmake -B build -DLLAMA_OPENSSL=OFF \
303
+ -DCMAKE_BUILD_TYPE=Release \
304
+ -DLLAMA_BUILD_EXAMPLES=ON \
305
+ -DGGML_CPU_REPACK=OFF \
306
+ -DLLAMA_BUILD_TOOLS=ON \
307
+ -DLLAMA_BUILD_TESTS=OFF \
308
+ -DGGML_CPU_RISCV64_SPACEMIT=ON \
309
+ -DGGML_RVV=ON \
310
+ -DGGML_RV_ZVFH=ON \
311
+ -DGGML_RV_ZFH=ON \
312
+ -DGGML_RV_ZICBOP=ON \
313
+ -DGGML_RV_ZIHINTPAUSE=ON \
314
+ -DGGML_RV_ZBA=ON \
315
+ -DCMAKE_TOOLCHAIN_FILE=${PWD}/cmake/riscv64-spacemit-linux-gnu-gcc.cmake
316
+
317
+ cmake --build build --config Release -j $(nproc)
.github/workflows/build-cuda-ubuntu.yml ADDED
@@ -0,0 +1,193 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: CI (CUDA, ubuntu)
2
+
3
+ on:
4
+ workflow_dispatch: # allows manual triggering
5
+ push:
6
+ branches:
7
+ - master
8
+ paths: [
9
+ '.github/workflows/build-cuda-ubuntu.yml',
10
+ '**/CMakeLists.txt',
11
+ '**/.cmake',
12
+ '**/*.h',
13
+ '**/*.hpp',
14
+ '**/*.c',
15
+ '**/*.cpp',
16
+ '**/*.cu',
17
+ '**/*.cuh'
18
+ ]
19
+
20
+ pull_request:
21
+ types: [opened, synchronize, reopened]
22
+ paths: [
23
+ '.github/workflows/build-cuda-ubuntu.yml',
24
+ 'ggml/src/ggml-cuda/**'
25
+ ]
26
+
27
+ concurrency:
28
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
29
+ cancel-in-progress: true
30
+
31
+ env:
32
+ GGML_NLOOP: 3
33
+ GGML_N_THREADS: 1
34
+ LLAMA_ARG_LOG_COLORS: 1
35
+ LLAMA_ARG_LOG_PREFIX: 1
36
+ LLAMA_ARG_LOG_TIMESTAMPS: 1
37
+
38
+ jobs:
39
+ cuda:
40
+ runs-on: ubuntu-24.04
41
+ container: nvidia/cuda:12.6.2-devel-ubuntu24.04
42
+
43
+ steps:
44
+ - name: Clone
45
+ id: checkout
46
+ uses: actions/checkout@v6
47
+
48
+ - name: Install dependencies
49
+ env:
50
+ DEBIAN_FRONTEND: noninteractive
51
+ run: |
52
+ apt update
53
+ apt install -y cmake build-essential ninja-build libgomp1 git libssl-dev jq python3 python3-venv python3-pip
54
+
55
+ - name: ccache
56
+ uses: ggml-org/ccache-action@v1.2.24
57
+ with:
58
+ key: cuda-ubuntu-24.04-cuda
59
+ save: false
60
+
61
+ - name: ccache-buckets-restore
62
+ uses: ./.github/actions/ccache-buckets
63
+ env:
64
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
65
+ with:
66
+ key: cuda-ubuntu-24.04-cuda
67
+ folder: llama.cpp
68
+ hf_bucket: ggml-org/cache
69
+
70
+ - name: Build with CMake
71
+ # TODO: Remove GGML_CUDA_CUB_3DOT2 flag once CCCL 3.2 is bundled within CTK and that CTK version is used in this project
72
+ run: |
73
+ cmake -S . -B build -G Ninja \
74
+ -DLLAMA_FATAL_WARNINGS=ON \
75
+ -DCMAKE_BUILD_TYPE=Release \
76
+ -DCMAKE_CUDA_ARCHITECTURES=89-real \
77
+ -DCMAKE_EXE_LINKER_FLAGS=-Wl,--allow-shlib-undefined \
78
+ -DGGML_NATIVE=OFF \
79
+ -DGGML_CUDA=ON \
80
+ -DGGML_CUDA_CUB_3DOT2=ON
81
+ cmake --build build
82
+
83
+ - name: ccache-buckets-save
84
+ if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
85
+ uses: ./.github/actions/ccache-buckets
86
+ env:
87
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
88
+ with:
89
+ key: cuda-ubuntu-24.04-cuda
90
+ folder: llama.cpp
91
+ evict-old-files: 1d
92
+ hf_bucket: ggml-org/cache
93
+ save: true
94
+
95
+ hip:
96
+ runs-on: ubuntu-22.04
97
+ container: rocm/dev-ubuntu-22.04:6.1.2
98
+
99
+ steps:
100
+ - name: Clone
101
+ id: checkout
102
+ uses: actions/checkout@v6
103
+
104
+ - name: Dependencies
105
+ id: depends
106
+ run: |
107
+ sudo apt-get update
108
+ sudo apt-get install -y build-essential git cmake rocblas-dev hipblas-dev libssl-dev rocwmma-dev jq python3-venv
109
+
110
+ - name: ccache
111
+ uses: ggml-org/ccache-action@v1.2.24
112
+ with:
113
+ key: cuda-ubuntu-22.04-hip
114
+ save: false
115
+
116
+ - name: ccache-buckets-restore
117
+ uses: ./.github/actions/ccache-buckets
118
+ env:
119
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
120
+ with:
121
+ key: cuda-ubuntu-22.04-hip
122
+ folder: llama.cpp
123
+ hf_bucket: ggml-org/cache
124
+
125
+ - name: Build with native CMake HIP support
126
+ id: cmake_build
127
+ run: |
128
+ cmake -B build -S . \
129
+ -DCMAKE_HIP_COMPILER="$(hipconfig -l)/clang" \
130
+ -DGPU_TARGETS="gfx1030" \
131
+ -DGGML_HIP=ON
132
+ cmake --build build --config Release -j $(nproc)
133
+
134
+ - name: ccache-buckets-save
135
+ if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
136
+ uses: ./.github/actions/ccache-buckets
137
+ env:
138
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
139
+ with:
140
+ key: cuda-ubuntu-22.04-hip
141
+ folder: llama.cpp
142
+ evict-old-files: 1d
143
+ hf_bucket: ggml-org/cache
144
+ save: true
145
+
146
+ musa:
147
+ runs-on: ubuntu-22.04
148
+ container: mthreads/musa:rc4.3.0-devel-ubuntu22.04-amd64
149
+
150
+ steps:
151
+ - name: Clone
152
+ id: checkout
153
+ uses: actions/checkout@v6
154
+
155
+ - name: Dependencies
156
+ id: depends
157
+ run: |
158
+ apt-get update
159
+ apt-get install -y build-essential git cmake libssl-dev jq
160
+
161
+ - name: ccache
162
+ uses: ggml-org/ccache-action@v1.2.24
163
+ with:
164
+ key: cuda-ubuntu-22.04-musa
165
+ save: false
166
+
167
+ - name: ccache-buckets-restore
168
+ uses: ./.github/actions/ccache-buckets
169
+ env:
170
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
171
+ with:
172
+ key: cuda-ubuntu-22.04-musa
173
+ folder: llama.cpp
174
+ hf_bucket: ggml-org/cache
175
+
176
+ - name: Build with native CMake MUSA support
177
+ id: cmake_build
178
+ run: |
179
+ cmake -B build -S . \
180
+ -DGGML_MUSA=ON
181
+ time cmake --build build --config Release -j $(nproc)
182
+
183
+ - name: ccache-buckets-save
184
+ if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
185
+ uses: ./.github/actions/ccache-buckets
186
+ env:
187
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
188
+ with:
189
+ key: cuda-ubuntu-22.04-musa
190
+ folder: llama.cpp
191
+ evict-old-files: 1d
192
+ hf_bucket: ggml-org/cache
193
+ save: true
.github/workflows/build-cuda-windows.yml ADDED
@@ -0,0 +1,183 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: CI (CUDA, windows)
2
+
3
+ # TODO: this workflow is only triggered manually because it is very heavy on the CI
4
+ # when we provision dedicated windows runners, we can enable it for pushes too
5
+ # note: running this workflow manually will populate the ccache for the release builds
6
+ # this can be used before merging a PR to speed up the release workflow
7
+ on:
8
+ workflow_dispatch: # allows manual triggering
9
+
10
+ # note: this will run in queue with the release workflow
11
+ concurrency:
12
+ group: release
13
+ queue: max
14
+
15
+ env:
16
+ GH_TOKEN: ${{ github.token }}
17
+ GGML_NLOOP: 3
18
+ GGML_N_THREADS: 1
19
+ LLAMA_ARG_LOG_COLORS: 1
20
+ LLAMA_ARG_LOG_PREFIX: 1
21
+ LLAMA_ARG_LOG_TIMESTAMPS: 1
22
+
23
+ jobs:
24
+ cuda:
25
+ name: windows-cuda (${{ matrix.cuda }}, ${{ matrix.arch }})
26
+ runs-on: windows-2022
27
+
28
+ permissions:
29
+ actions: write
30
+
31
+ strategy:
32
+ matrix:
33
+ include:
34
+ - cuda: '12.4'
35
+ arch: x64
36
+ defines: '-DGGML_CUDA_CUB_3DOT2=ON'
37
+ - cuda: '13.3'
38
+ arch: x64
39
+ defines: ''
40
+ - cuda: '13.4'
41
+ arch: arm64
42
+ defines: '-DCMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc-cuda.cmake'
43
+
44
+ steps:
45
+ - name: Clone
46
+ id: checkout
47
+ uses: actions/checkout@v6
48
+
49
+ - name: ccache
50
+ uses: ggml-org/ccache-action@v1.2.24
51
+ with:
52
+ key: release-windows-2022-${{ matrix.arch }}-cuda-${{ matrix.cuda }}
53
+
54
+ - name: Install Cuda Toolkit
55
+ uses: ./.github/actions/windows-setup-cuda
56
+ with:
57
+ cuda_version: ${{ matrix.cuda }}
58
+ cuda_arch: ${{ matrix.arch }}
59
+
60
+ - name: Install Ninja
61
+ id: install_ninja
62
+ run: |
63
+ choco install ninja
64
+
65
+ - name: Build
66
+ id: cmake_build
67
+ shell: cmd
68
+ run: |
69
+ call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.arch == 'x64' && 'x64' || 'amd64_arm64' }}
70
+ cmake -S . -B build -G "Ninja Multi-Config" ^
71
+ -DGGML_BACKEND_DL=ON ^
72
+ -DGGML_NATIVE=OFF ^
73
+ -DGGML_CPU=OFF ^
74
+ -DGGML_CUDA=ON ^
75
+ -DLLAMA_BUILD_BORINGSSL=ON ${{ matrix.defines }}
76
+ set /A NINJA_JOBS=%NUMBER_OF_PROCESSORS%-1
77
+ cmake --build build --config Release -j %NINJA_JOBS% --target ggml-cuda
78
+
79
+ - name: ccache-clear
80
+ uses: ./.github/actions/ccache-clear
81
+ with:
82
+ key: release-windows-2022-${{ matrix.arch }}-cuda-${{ matrix.cuda }}
83
+
84
+ hip:
85
+ runs-on: windows-2022
86
+
87
+ permissions:
88
+ actions: write
89
+
90
+ env:
91
+ # Make sure this is in sync with build-cache.yml
92
+ ROCM_VERSION: "7.14.0"
93
+
94
+ strategy:
95
+ matrix:
96
+ include:
97
+ # sync with release.yml
98
+ - name: "radeon"
99
+ gpu_targets: "gfx1150;gfx1151;gfx1200;gfx1201;gfx1100;gfx1101;gfx1102;gfx1030;gfx1031;gfx1032"
100
+
101
+ steps:
102
+ - name: Clone
103
+ id: checkout
104
+ uses: actions/checkout@v6
105
+
106
+ # - name: Cache ROCm Installation
107
+ # uses: actions/cache@v5
108
+ # id: cache-rocm
109
+ # with:
110
+ # path: C:\TheRock\build
111
+ # key: rocm-wheels-${{ env.ROCM_VERSION }}-multi-arch-${{ runner.os }}
112
+
113
+ - name: Setup ROCm
114
+ # if: steps.cache-rocm.outputs.cache-hit != 'true'
115
+ uses: ./.github/actions/windows-setup-rocm
116
+ with:
117
+ version: ${{ env.ROCM_VERSION }}
118
+
119
+ - name: Setup ROCm Environment
120
+ run: |
121
+ $ErrorActionPreference = "Stop"
122
+
123
+ # Activate venv from cache or fresh install
124
+ & C:\TheRock\build\.venv\Scripts\Activate.ps1
125
+
126
+ # Expand the devel tree (idempotent; no-op if already done during install)
127
+ rocm-sdk init
128
+ if ($LASTEXITCODE -ne 0) { throw "rocm-sdk init failed with exit code $LASTEXITCODE" }
129
+
130
+ # Get ROCm installation paths using the rocm-sdk CLI tool
131
+ $rocmPath = (rocm-sdk path --root)
132
+ if (-not $rocmPath) { throw "rocm-sdk path --root returned empty - devel package may not be installed" }
133
+ $rocmPath = $rocmPath.Trim()
134
+ $cmakePath = (rocm-sdk path --cmake).Trim()
135
+ $binPath = (rocm-sdk path --bin).Trim()
136
+ write-host "ROCm root: $rocmPath"
137
+
138
+ echo "HIP_PATH=$rocmPath" >> $env:GITHUB_ENV
139
+ echo "CMAKE_PREFIX_PATH=$cmakePath" >> $env:GITHUB_ENV
140
+ echo "HIP_DEVICE_LIB_PATH=$rocmPath\lib\llvm\amdgcn\bitcode" >> $env:GITHUB_ENV
141
+ echo "HIP_PLATFORM=amd" >> $env:GITHUB_ENV
142
+ echo "LLVM_PATH=$rocmPath\lib\llvm" >> $env:GITHUB_ENV
143
+ echo "$binPath" >> $env:GITHUB_PATH
144
+
145
+ # Keep venv in PATH for subsequent steps
146
+ echo "C:\TheRock\build\.venv\Scripts" >> $env:GITHUB_PATH
147
+
148
+ - name: Verify ROCm
149
+ id: verify
150
+ run: |
151
+ # Test the ROCm clang shipped in the installed wheel
152
+ & "${env:HIP_PATH}\lib\llvm\bin\clang.exe" --version
153
+
154
+ - name: ccache
155
+ uses: ggml-org/ccache-action@v1.2.24
156
+ with:
157
+ # TODO: this build does not match the build in release.yml, so we use a different cache key
158
+ # ideally, the builds should match, similar to the CUDA build above so that we would be able
159
+ # to populate the ccache for the release with manual runs of this workflow
160
+ #key: release-windows-2022-x64-hip-${{ env.ROCM_VERSION }}-${{ matrix.name }}
161
+ key: cuda-windows-2022-x64-hip-${{ env.ROCM_VERSION }}-${{ matrix.name }}
162
+
163
+ - name: Build
164
+ id: cmake_build
165
+ run: |
166
+ cmake -G "Unix Makefiles" -B build -S . `
167
+ -DCMAKE_PREFIX_PATH="${env:HIP_PATH}" `
168
+ -DCMAKE_C_COMPILER="${env:HIP_PATH}\lib\llvm\bin\clang.exe" `
169
+ -DCMAKE_CXX_COMPILER="${env:HIP_PATH}\lib\llvm\bin\clang++.exe" `
170
+ -DCMAKE_HIP_COMPILER="${env:HIP_PATH}\lib\llvm\bin\clang.exe" `
171
+ -DCMAKE_BUILD_TYPE=Release `
172
+ -DLLAMA_BUILD_BORINGSSL=ON `
173
+ -DHIP_PATH="${env:HIP_PATH}" `
174
+ -DGGML_HIP=ON `
175
+ -DGPU_TARGETS="gfx1100" `
176
+ -DGGML_RPC=ON
177
+ cmake --build build -j ${env:NUMBER_OF_PROCESSORS}
178
+
179
+ - name: ccache-clear
180
+ uses: ./.github/actions/ccache-clear
181
+ with:
182
+ #key: release-windows-2022-x64-hip-${{ env.ROCM_VERSION }}-${{ matrix.name }}
183
+ key: cuda-windows-2022-x64-hip-${{ env.ROCM_VERSION }}-${{ matrix.name }}
.github/workflows/build-ibm.yml ADDED
@@ -0,0 +1,150 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: CI (ibm)
2
+
3
+ on:
4
+ workflow_dispatch: # allows manual triggering
5
+ push:
6
+ branches:
7
+ - master
8
+ paths: [
9
+ '.github/workflows/build-ibm.yml',
10
+ '**/CMakeLists.txt',
11
+ '**/.cmake',
12
+ '**/*.h',
13
+ '**/*.hpp',
14
+ '**/*.c',
15
+ '**/*.cpp'
16
+ ]
17
+
18
+ pull_request:
19
+ types: [opened, synchronize, reopened]
20
+ paths: [
21
+ '.github/workflows/build-ibm.yml',
22
+ 'ggml/src/ggml-cpu/**'
23
+ ]
24
+
25
+ concurrency:
26
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
27
+ cancel-in-progress: true
28
+
29
+ env:
30
+ GGML_NLOOP: 3
31
+ GGML_N_THREADS: 1
32
+ LLAMA_ARG_LOG_COLORS: 1
33
+ LLAMA_ARG_LOG_PREFIX: 1
34
+ LLAMA_ARG_LOG_TIMESTAMPS: 1
35
+
36
+ jobs:
37
+
38
+ ubuntu-24-s390x:
39
+ runs-on: ubuntu-24.04-s390x
40
+
41
+ steps:
42
+ - name: Clone
43
+ id: checkout
44
+ uses: actions/checkout@v6
45
+
46
+ - name: Build Dependencies
47
+ id: build_depends
48
+ run: |
49
+ sudo apt-get update
50
+ sudo apt-get install -y --no-install-recommends \
51
+ python3 python3-pip python3-dev python3-wheel \
52
+ libjpeg-dev build-essential libssl-dev \
53
+ git-lfs
54
+
55
+ - name: Toolchain workaround (GCC 14)
56
+ run: |
57
+ sudo apt-get install -y gcc-14 g++-14
58
+ echo "CC=gcc-14" >> "$GITHUB_ENV"
59
+ echo "CXX=g++-14" >> "$GITHUB_ENV"
60
+
61
+ - name: Python Dependencies
62
+ id: python_depends
63
+ run: |
64
+ export PIP_BREAK_SYSTEM_PACKAGES="1"
65
+ python3 -m pip install --upgrade pip setuptools
66
+ pip3 install ./gguf-py
67
+
68
+ - name: Swap Endianness
69
+ id: endianness
70
+ run: |
71
+ for f in models/*.gguf; do
72
+ echo YES | python3 gguf-py/gguf/scripts/gguf_convert_endian.py $f big
73
+ done
74
+
75
+ - name: Build
76
+ id: cmake_build
77
+ run: |
78
+ cmake -B build \
79
+ -DLLAMA_FATAL_WARNINGS=ON \
80
+ -DGGML_RPC=ON
81
+ time cmake --build build --config Release -j $(nproc)
82
+
83
+ - name: Test
84
+ id: cmake_test
85
+ run: |
86
+ cd build
87
+ ctest -L main --verbose --timeout 900
88
+
89
+ - name: Test llama2c (s390x)
90
+ id: llama2c_test_s390x
91
+ run: |
92
+ cd build
93
+ echo "Fetch llama2c big-endian model"
94
+ wget https://huggingface.co/ggml-org/models/resolve/main/tinyllamas/stories260K-be.gguf
95
+ ./bin/llama-completion -m stories260K-be.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256
96
+
97
+ ubuntu-24-ppc64le:
98
+ runs-on: ubuntu-24.04-ppc64le
99
+
100
+ steps:
101
+ - name: Clone
102
+ id: checkout
103
+ uses: actions/checkout@v6
104
+
105
+ - name: Build Dependencies
106
+ id: build_depends
107
+ run: |
108
+ sudo apt-get update
109
+ sudo apt-get install -y --no-install-recommends \
110
+ python3 python3-pip python3-dev python3-wheel \
111
+ libjpeg-dev build-essential libssl-dev \
112
+ git-lfs
113
+
114
+ - name: Toolchain workaround (GCC 14)
115
+ run: |
116
+ sudo apt-get install -y gcc-14 g++-14
117
+ echo "CC=gcc-14" >> "$GITHUB_ENV"
118
+ echo "CXX=g++-14" >> "$GITHUB_ENV"
119
+
120
+ - name: Python Dependencies
121
+ id: python_depends
122
+ run: |
123
+ export PIP_BREAK_SYSTEM_PACKAGES="1"
124
+ python3 -m pip install --upgrade pip setuptools
125
+ pip3 install ./gguf-py
126
+
127
+ - name: Build
128
+ id: cmake_build
129
+ run: |
130
+ cmake -B build \
131
+ -DLLAMA_FATAL_WARNINGS=ON \
132
+ -DGGML_RPC=ON
133
+ time cmake --build build --config Release -j $(nproc)
134
+
135
+ - name: Test
136
+ id: cmake_test
137
+ run: |
138
+ cd build
139
+ ctest -L main --verbose --timeout 900
140
+
141
+ - name: Test llama2c conversion
142
+ id: llama2c_test
143
+ run: |
144
+ cd build
145
+ echo "Fetch tokenizer"
146
+ wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/tok512.bin
147
+ echo "Fetch llama2c model"
148
+ wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/stories260K.bin
149
+ ./bin/llama-convert-llama2c-to-ggml --copy-vocab-from-model ./tok512.bin --llama2c-model stories260K.bin --llama2c-output-model stories260K.gguf
150
+ ./bin/llama-completion -m stories260K.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256
.github/workflows/build-msys.yml ADDED
@@ -0,0 +1,70 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: CI (msys)
2
+
3
+ on:
4
+ # only manual triggers due to low-importance of the workflows
5
+ # TODO: for regular runs, provision dedicated self-hosted runners
6
+ workflow_dispatch:
7
+ # run once every week
8
+ schedule:
9
+ - cron: '0 0 * * 0'
10
+
11
+ concurrency:
12
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
13
+ cancel-in-progress: true
14
+
15
+ env:
16
+ GGML_NLOOP: 3
17
+ GGML_N_THREADS: 1
18
+ LLAMA_ARG_LOG_COLORS: 1
19
+ LLAMA_ARG_LOG_PREFIX: 1
20
+ LLAMA_ARG_LOG_TIMESTAMPS: 1
21
+
22
+ jobs:
23
+ windows-msys2:
24
+ runs-on: windows-2025
25
+
26
+ strategy:
27
+ fail-fast: false
28
+ matrix:
29
+ include:
30
+ - { sys: UCRT64, env: ucrt-x86_64, compiler: gcc, build: Release }
31
+ - { sys: CLANG64, env: clang-x86_64, compiler: clang, build: Release }
32
+
33
+ steps:
34
+ - name: Clone
35
+ uses: actions/checkout@v6
36
+
37
+ #- name: ccache
38
+ # uses: ggml-org/ccache-action@v1.2.24
39
+ # with:
40
+ # key: msys-windows-2025-x64
41
+ # variant: ccache
42
+ # evict-old-files: 1d
43
+ # save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
44
+
45
+ - name: Setup ${{ matrix.sys }}
46
+ uses: msys2/setup-msys2@cafece8e6baf9247cf9b1bf95097b0b983cc558d # v2
47
+ with:
48
+ update: true
49
+ msystem: ${{matrix.sys}}
50
+ install: >-
51
+ mingw-w64-${{matrix.env}}-${{matrix.compiler}}
52
+ mingw-w64-${{matrix.env}}-cmake
53
+ mingw-w64-${{matrix.env}}-openblas
54
+
55
+ - name: Build using CMake
56
+ shell: msys2 {0}
57
+ run: |
58
+ cmake -B build
59
+ cmake --build build --config ${{ matrix.build }} -j $(nproc)
60
+
61
+ - name: Clean after building using CMake
62
+ shell: msys2 {0}
63
+ run: |
64
+ rm -rf build
65
+
66
+ - name: Build using CMake w/ OpenBLAS
67
+ shell: msys2 {0}
68
+ run: |
69
+ cmake -B build -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS
70
+ cmake --build build --config ${{ matrix.build }} -j $(nproc)
.github/workflows/build-opencl.yml ADDED
@@ -0,0 +1,92 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: CI (opencl)
2
+
3
+ on:
4
+ workflow_dispatch: # allows manual triggering
5
+ push:
6
+ branches:
7
+ - master
8
+ paths: [
9
+ '.github/workflows/build-opencl.yml',
10
+ '**/CMakeLists.txt',
11
+ '**/.cmake',
12
+ '**/*.h',
13
+ '**/*.hpp',
14
+ '**/*.c',
15
+ '**/*.cpp',
16
+ '**/*.cl'
17
+ ]
18
+
19
+ pull_request:
20
+ types: [opened, synchronize, reopened]
21
+ paths: [
22
+ '.github/workflows/build-opencl.yml',
23
+ 'ggml/src/ggml-opencl/**'
24
+ ]
25
+
26
+ concurrency:
27
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
28
+ cancel-in-progress: true
29
+
30
+ env:
31
+ GGML_NLOOP: 3
32
+ GGML_N_THREADS: 1
33
+ LLAMA_ARG_LOG_COLORS: 1
34
+ LLAMA_ARG_LOG_PREFIX: 1
35
+ LLAMA_ARG_LOG_TIMESTAMPS: 1
36
+
37
+ jobs:
38
+ windows-2025-opencl-adreno:
39
+ runs-on: windows-2025
40
+
41
+ steps:
42
+ - name: Clone
43
+ id: checkout
44
+ uses: actions/checkout@v6
45
+
46
+ - name: ccache
47
+ uses: ggml-org/ccache-action@v1.2.24
48
+ with:
49
+ key: opencl-windows-2025-x64
50
+ variant: ccache
51
+ evict-old-files: 1d
52
+ save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
53
+
54
+ - name: Install Ninja
55
+ id: install_ninja
56
+ run: |
57
+ choco install ninja
58
+
59
+ - name: Install OpenCL Headers and Libs
60
+ id: install_opencl
61
+ run: |
62
+ git clone https://github.com/KhronosGroup/OpenCL-Headers
63
+ cd OpenCL-Headers
64
+ cmake -B build `
65
+ -DBUILD_TESTING=OFF `
66
+ -DOPENCL_HEADERS_BUILD_TESTING=OFF `
67
+ -DOPENCL_HEADERS_BUILD_CXX_TESTS=OFF `
68
+ -DCMAKE_INSTALL_PREFIX="$env:RUNNER_TEMP/opencl-arm64-release"
69
+ cmake --build build --target install
70
+ git clone https://github.com/KhronosGroup/OpenCL-ICD-Loader
71
+ cd OpenCL-ICD-Loader
72
+ cmake -B build-arm64-release `
73
+ -A arm64 `
74
+ -DCMAKE_PREFIX_PATH="$env:RUNNER_TEMP/opencl-arm64-release" `
75
+ -DCMAKE_INSTALL_PREFIX="$env:RUNNER_TEMP/opencl-arm64-release"
76
+ cmake --build build-arm64-release --target install --config release
77
+
78
+ - name: Build
79
+ id: cmake_build
80
+ run: |
81
+ cmake -S . -B build -G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DCMAKE_PREFIX_PATH="$env:RUNNER_TEMP/opencl-arm64-release" -DGGML_OPENCL=ON -DGGML_OPENCL_USE_ADRENO_KERNELS=ON -DLLAMA_BUILD_BORINGSSL=ON
82
+ cmake --build build --config Release -j ${env:NUMBER_OF_PROCESSORS}
83
+
84
+ - name: ccache-clear
85
+ uses: ./.github/actions/ccache-clear
86
+ env:
87
+ GH_TOKEN: ${{ github.token }}
88
+ with:
89
+ key: opencl-windows-2025-x64
90
+ older: 5m
91
+ min: 1
92
+ dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
.github/workflows/build-openvino.yml ADDED
@@ -0,0 +1,178 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: CI (openvino)
2
+
3
+ on:
4
+ workflow_dispatch: # allows manual triggering
5
+ push:
6
+ branches:
7
+ - master
8
+ paths: [
9
+ '.github/workflows/build-openvino.yml',
10
+ '**/CMakeLists.txt',
11
+ '**/.cmake',
12
+ '**/*.h',
13
+ '**/*.hpp',
14
+ '**/*.c',
15
+ '**/*.cpp',
16
+ ]
17
+
18
+ pull_request:
19
+ types: [opened, synchronize, reopened]
20
+ paths: [
21
+ '.github/workflows/build-openvino.yml',
22
+ 'ggml/src/ggml-openvino/**'
23
+ ]
24
+
25
+ concurrency:
26
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
27
+ cancel-in-progress: true
28
+
29
+ env:
30
+ GGML_NLOOP: 3
31
+ GGML_N_THREADS: 1
32
+ LLAMA_ARG_LOG_COLORS: 1
33
+ LLAMA_ARG_LOG_PREFIX: 1
34
+ LLAMA_ARG_LOG_TIMESTAMPS: 1
35
+ # TODO: fix failing tests on OpenVINO backend
36
+ CTEST_EXCLUDE: "test-llama-archs|^test-recurrent-state-|test-backend-ops|test-save-load-state"
37
+
38
+ jobs:
39
+ ubuntu-24-openvino:
40
+ runs-on: [self-hosted, Linux, Intel, OpenVINO]
41
+
42
+ env:
43
+ # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
44
+ OPENVINO_VERSION_MAJOR: "2026.3.1"
45
+ OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
46
+
47
+ steps:
48
+ - name: Clone
49
+ id: checkout
50
+ uses: actions/checkout@v6
51
+
52
+ - name: Dependencies
53
+ id: depends
54
+ run: |
55
+ sudo apt-get update
56
+ sudo apt-get install -y build-essential libssl-dev libtbb12 cmake ninja-build python3-pip
57
+ sudo apt-get install -y ocl-icd-opencl-dev opencl-headers opencl-clhpp-headers intel-opencl-icd
58
+
59
+ - name: Setup OpenVINO Toolkit
60
+ uses: ./.github/actions/linux-setup-openvino
61
+ with:
62
+ path: ./openvino_toolkit
63
+ version_major: ${{ env.OPENVINO_VERSION_MAJOR }}
64
+ version_full: ${{ env.OPENVINO_VERSION_FULL }}
65
+
66
+ - name: Install OpenVINO dependencies
67
+ run: |
68
+ cd ./openvino_toolkit
69
+ chmod +x ./install_dependencies/install_openvino_dependencies.sh
70
+ echo "Y" | sudo -E ./install_dependencies/install_openvino_dependencies.sh
71
+
72
+ - name: Build
73
+ id: cmake_build
74
+ run: |
75
+ source ./openvino_toolkit/setupvars.sh
76
+ cmake -B build/ReleaseOV -G Ninja \
77
+ -DCMAKE_BUILD_TYPE=Release \
78
+ -DGGML_OPENVINO=ON
79
+ time cmake --build build/ReleaseOV --config Release --parallel
80
+
81
+ - name: Test (CPU)
82
+ id: cmake_test_cpu
83
+ run: |
84
+ cd ${{ github.workspace }}
85
+ ctest --test-dir build/ReleaseOV -L main -E "${{ env.CTEST_EXCLUDE }}" --verbose --timeout 3000
86
+
87
+ - name: Test (GPU)
88
+ id: cmake_test_gpu
89
+ run: |
90
+ cd ${{ github.workspace }}
91
+ export GGML_OPENVINO_DEVICE=GPU
92
+ ctest --test-dir build/ReleaseOV -L main -E "${{ env.CTEST_EXCLUDE }}" --verbose --timeout 3000
93
+
94
+ openvino-windows-2022:
95
+ runs-on: windows-2022
96
+
97
+ env:
98
+ # Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
99
+ OPENVINO_VERSION_MAJOR: "2026.3.1"
100
+ OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
101
+
102
+ steps:
103
+ - name: Clone
104
+ id: checkout
105
+ uses: actions/checkout@v6
106
+
107
+ - name: ccache
108
+ uses: ggml-org/ccache-action@v1.2.24
109
+ with:
110
+ key: openvino-windows-2022
111
+ variant: ccache
112
+ evict-old-files: 1d
113
+ save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
114
+
115
+ - name: Setup Cache
116
+ uses: actions/cache@v5
117
+ id: cache-openvino
118
+ with:
119
+ path: ./openvino_toolkit
120
+ key: cache-gha-openvino-toolkit-v${{ env.OPENVINO_VERSION_FULL }}-${{ runner.os }}
121
+
122
+ - name: Setup OpenVINO Toolkit
123
+ if: steps.cache-openvino.outputs.cache-hit != 'true'
124
+ uses: ./.github/actions/windows-setup-openvino
125
+ with:
126
+ path: ./openvino_toolkit
127
+ version_major: ${{ env.OPENVINO_VERSION_MAJOR }}
128
+ version_full: ${{ env.OPENVINO_VERSION_FULL }}
129
+
130
+ - name: Install OpenCL using vcpkg
131
+ shell: powershell
132
+ run: |
133
+ git clone https://github.com/microsoft/vcpkg C:\vcpkg
134
+ C:\vcpkg\bootstrap-vcpkg.bat
135
+ C:\vcpkg\vcpkg install opencl
136
+
137
+ - name: Build
138
+ id: cmake_build
139
+ shell: cmd
140
+ run: |
141
+ REM Find extracted OpenVINO folder dynamically
142
+ for /d %%i in (openvino_toolkit\*) do set OPENVINO_ROOT=%%i
143
+
144
+ if not exist "%OPENVINO_ROOT%\runtime\cmake\OpenVINOConfig.cmake" (
145
+ echo ERROR: OpenVINOConfig.cmake not found
146
+ exit /b 1
147
+ )
148
+
149
+ call "%OPENVINO_ROOT%\setupvars.bat"
150
+
151
+ cmake -B build\ReleaseOV -G "Visual Studio 17 2022" ^
152
+ -A x64 ^
153
+ -DCMAKE_BUILD_TYPE=Release ^
154
+ -DGGML_OPENVINO=ON ^
155
+ -DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake
156
+
157
+ cmake --build build\ReleaseOV --config Release -- /m
158
+
159
+ - name: Test (CPU)
160
+ id: cmake_test_cpu
161
+ shell: cmd
162
+ run: |
163
+ REM Find extracted OpenVINO folder dynamically
164
+ for /d %%i in (openvino_toolkit\*) do set OPENVINO_ROOT=%%i
165
+ call "%OPENVINO_ROOT%\setupvars.bat"
166
+
167
+ cd build
168
+ ctest --test-dir ReleaseOV -L main -E "${{ env.CTEST_EXCLUDE }}" -C Release --verbose --timeout 3000
169
+
170
+ - name: ccache-clear
171
+ uses: ./.github/actions/ccache-clear
172
+ env:
173
+ GH_TOKEN: ${{ github.token }}
174
+ with:
175
+ key: openvino-windows-2022
176
+ older: 5m
177
+ min: 1
178
+ dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
.github/workflows/build-riscv.yml ADDED
@@ -0,0 +1,188 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: CI (riscv)
2
+
3
+ on:
4
+ workflow_dispatch: # allows manual triggering
5
+ push:
6
+ branches:
7
+ - master
8
+ paths: [
9
+ '.github/workflows/build-riscv.yml',
10
+ '**/CMakeLists.txt',
11
+ '**/.cmake',
12
+ '**/*.h',
13
+ '**/*.hpp',
14
+ '**/*.c',
15
+ '**/*.cpp'
16
+ ]
17
+
18
+ pull_request:
19
+ types: [opened, synchronize, reopened]
20
+ paths: [
21
+ '.github/workflows/build-riscv.yml',
22
+ 'ggml/src/ggml-cpu/arch/riscv/**'
23
+ ]
24
+
25
+ concurrency:
26
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
27
+ cancel-in-progress: true
28
+
29
+ env:
30
+ GGML_NLOOP: 3
31
+ GGML_N_THREADS: 1
32
+ LLAMA_ARG_LOG_COLORS: 1
33
+ LLAMA_ARG_LOG_PREFIX: 1
34
+ LLAMA_ARG_LOG_TIMESTAMPS: 1
35
+
36
+ jobs:
37
+ ubuntu-cpu-riscv64-native:
38
+ runs-on: ubuntu-24.04-riscv
39
+
40
+ steps:
41
+ - name: Install dependencies
42
+ run: |
43
+ # Install necessary packages
44
+ sudo apt-get update
45
+ sudo apt-get install -y libssl-dev
46
+
47
+ # Set gcc-14 and g++-14 as the default compilers
48
+ sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 100
49
+ sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-14 100
50
+
51
+ git lfs install
52
+
53
+ - name: Check environment
54
+ run: |
55
+ uname -a
56
+ gcc --version
57
+ g++ --version
58
+ ldd --version
59
+ cmake --version
60
+ rustc --version
61
+ env
62
+ echo "nproc=$(nproc)"
63
+
64
+ - name: Clone
65
+ id: checkout
66
+ uses: actions/checkout@v6
67
+
68
+ # note: sparing some ccache since these jobs run on dedicated runners that are not part of the organitzation
69
+ #- name: ccache
70
+ # uses: ggml-org/ccache-action@v1.2.24
71
+ # with:
72
+ # key: riscv-ubuntu-native
73
+ # evict-old-files: 1d
74
+ # save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
75
+
76
+ - name: Build
77
+ id: cmake_build
78
+ run: |
79
+ cmake -B build \
80
+ -DCMAKE_BUILD_TYPE=Release \
81
+ -DGGML_OPENMP=OFF \
82
+ -DLLAMA_BUILD_EXAMPLES=ON \
83
+ -DLLAMA_BUILD_TOOLS=ON \
84
+ -DLLAMA_BUILD_TESTS=ON \
85
+ -DCMAKE_C_COMPILER_LAUNCHER=ccache \
86
+ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
87
+ -DGGML_RPC=ON \
88
+ -DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \
89
+ -DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14
90
+
91
+ time cmake --build build --config Release -j $(nproc)
92
+
93
+ - name: Test
94
+ id: cmake_test
95
+ run: |
96
+ cd build
97
+ ctest -L main --verbose --timeout 900
98
+
99
+ - name: Test llama2c conversion
100
+ id: llama2c_test
101
+ run: |
102
+ cd build
103
+ echo "Fetch tokenizer"
104
+ wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/tok512.bin
105
+ echo "Fetch llama2c model"
106
+ wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/stories260K.bin
107
+ ./bin/llama-convert-llama2c-to-ggml --copy-vocab-from-model ./tok512.bin --llama2c-model stories260K.bin --llama2c-output-model stories260K.gguf
108
+ ./bin/llama-completion -m stories260K.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256
109
+
110
+ ubuntu-riscv64-native-sanitizer:
111
+ runs-on: ubuntu-24.04-riscv
112
+
113
+ continue-on-error: true
114
+
115
+ strategy:
116
+ matrix:
117
+ sanitizer: [ADDRESS, THREAD, UNDEFINED]
118
+ build_type: [Debug]
119
+
120
+ steps:
121
+ - name: Install dependencies
122
+ run: |
123
+ # Set gcc-14 and g++-14 as the default compilers
124
+ sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 100
125
+ sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-14 100
126
+
127
+ git lfs install
128
+
129
+ - name: GCC version check
130
+ run: |
131
+ gcc --version
132
+ g++ --version
133
+
134
+ - name: Clone
135
+ id: checkout
136
+ uses: actions/checkout@v6
137
+
138
+ # note: sparing some ccache since these jobs run on dedicated runners that are not part of the organitzation
139
+ #- name: ccache
140
+ # uses: ggml-org/ccache-action@v1.2.24
141
+ # with:
142
+ # key: riscv-ubuntu-native-sanitizer-${{ matrix.sanitizer }}-${{ matrix.build_type }}
143
+ # evict-old-files: 1d
144
+ # save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
145
+
146
+ - name: Build
147
+ id: cmake_build
148
+ if: ${{ matrix.sanitizer != 'THREAD' }}
149
+ run: |
150
+ cmake -B build \
151
+ -DLLAMA_OPENSSL=OFF \
152
+ -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
153
+ -DGGML_OPENMP=ON \
154
+ -DLLAMA_BUILD_EXAMPLES=ON \
155
+ -DLLAMA_BUILD_TOOLS=ON \
156
+ -DLLAMA_BUILD_TESTS=OFF \
157
+ -DCMAKE_C_COMPILER_LAUNCHER=ccache \
158
+ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
159
+ -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \
160
+ -DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \
161
+ -DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14
162
+
163
+ cmake --build build --config ${{ matrix.build_type }} -j $(nproc)
164
+
165
+ - name: Build (no OpenMP)
166
+ id: cmake_build_no_openmp
167
+ if: ${{ matrix.sanitizer == 'THREAD' }}
168
+ run: |
169
+ cmake -B build \
170
+ -DLLAMA_OPENSSL=OFF \
171
+ -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
172
+ -DGGML_OPENMP=OFF \
173
+ -DLLAMA_BUILD_EXAMPLES=ON \
174
+ -DLLAMA_BUILD_TOOLS=ON \
175
+ -DLLAMA_BUILD_TESTS=OFF \
176
+ -DCMAKE_C_COMPILER_LAUNCHER=ccache \
177
+ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
178
+ -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \
179
+ -DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \
180
+ -DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14
181
+
182
+ cmake --build build --config ${{ matrix.build_type }} -j $(nproc)
183
+
184
+ - name: Test
185
+ id: cmake_test
186
+ run: |
187
+ cd build
188
+ ctest -L main --verbose --timeout 900
.github/workflows/build-sanitize.yml ADDED
@@ -0,0 +1,108 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: CI (sanitize)
2
+
3
+ on:
4
+ workflow_dispatch: # allows manual triggering
5
+ push:
6
+ branches:
7
+ - master
8
+ paths: [
9
+ '.github/workflows/build-sanitize.yml',
10
+ '**/CMakeLists.txt',
11
+ '**/.cmake',
12
+ '**/*.h',
13
+ '**/*.hpp',
14
+ '**/*.c',
15
+ '**/*.cpp'
16
+ ]
17
+
18
+ pull_request:
19
+ types: [opened, synchronize, reopened]
20
+ paths: [
21
+ '.github/workflows/build-sanitize.yml'
22
+ ]
23
+
24
+ concurrency:
25
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
26
+ cancel-in-progress: true
27
+
28
+ env:
29
+ GGML_NLOOP: 3
30
+ GGML_N_THREADS: 1
31
+ LLAMA_ARG_LOG_COLORS: 1
32
+ LLAMA_ARG_LOG_PREFIX: 1
33
+ LLAMA_ARG_LOG_TIMESTAMPS: 1
34
+
35
+ jobs:
36
+ ctest:
37
+ continue-on-error: true
38
+
39
+ strategy:
40
+ matrix:
41
+ include:
42
+ # thread and address doesn't run properly on some self hosted machines, so run it on Github instead
43
+ - sanitizer: ADDRESS
44
+ machine: ubuntu-24.04
45
+ - sanitizer: THREAD
46
+ machine: ubuntu-24.04
47
+ - sanitizer: UNDEFINED
48
+ machine: [self-hosted, X64, Linux]
49
+
50
+ runs-on: ${{ matrix.machine }}
51
+
52
+ steps:
53
+ - name: Clone
54
+ id: checkout
55
+ uses: actions/checkout@v6
56
+
57
+ # - name: ccache
58
+ # uses: ggml-org/ccache-action@v1.2.24
59
+ # if: ${{ matrix.sanitizer != 'UNDEFINED' }}
60
+ # with:
61
+ # key: ctest-${{ matrix.sanitizer }}-ubuntu-24.04
62
+ # variant: ccache
63
+ # evict-old-files: 1d
64
+ # save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
65
+
66
+ # with UNDEFINED sanitizer, we have to build in Debug to avoid GCC 13 false-positive warnings
67
+ - name: Build (undefined)
68
+ id: cmake_build_undefined
69
+ if: ${{ matrix.sanitizer == 'UNDEFINED' }}
70
+ run: |
71
+ cmake -B build \
72
+ -DCMAKE_BUILD_TYPE=Debug \
73
+ -DLLAMA_FATAL_WARNINGS=ON \
74
+ -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \
75
+ -DGGML_SANITIZE_${{ matrix.sanitizer }}=ON
76
+
77
+ cmake --build build --config Debug -j $(nproc)
78
+
79
+ - name: Build
80
+ id: cmake_build
81
+ if: ${{ matrix.sanitizer == 'ADDRESS' }}
82
+ run: |
83
+ cmake -B build \
84
+ -DCMAKE_BUILD_TYPE=RelWithDebInfo \
85
+ -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \
86
+ -DGGML_SANITIZE_${{ matrix.sanitizer }}=ON
87
+
88
+ cmake --build build --config RelWithDebInfo -j $(nproc)
89
+
90
+ - name: Build (no OpenMP)
91
+ id: cmake_build_no_openmp
92
+ if: ${{ matrix.sanitizer == 'THREAD' }}
93
+ run: |
94
+ cmake -B build \
95
+ -DCMAKE_BUILD_TYPE=RelWithDebInfo \
96
+ -DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \
97
+ -DGGML_SANITIZE_${{ matrix.sanitizer }}=ON \
98
+ -DGGML_OPENMP=OFF
99
+
100
+ cmake --build build --config RelWithDebInfo -j $(nproc)
101
+
102
+ - name: Test
103
+ id: cmake_test
104
+ # skip run in Debug - very slow
105
+ if: ${{ matrix.sanitizer != 'UNDEFINED' }}
106
+ run: |
107
+ cd build
108
+ ctest -L main -E tokenizer --verbose --timeout 900
.github/workflows/build-self-hosted.yml ADDED
@@ -0,0 +1,409 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: CI (self-hosted)
2
+
3
+ on:
4
+ workflow_dispatch: # allows manual triggering
5
+ push:
6
+ branches:
7
+ - master
8
+ paths: [
9
+ '.github/workflows/build-self-hosted.yml',
10
+ '**/CMakeLists.txt',
11
+ '**/.cmake',
12
+ '**/*.h',
13
+ '**/*.hpp',
14
+ '**/*.c',
15
+ '**/*.cpp',
16
+ '**/*.cu',
17
+ '**/*.cuh',
18
+ '**/*.swift',
19
+ '**/*.m',
20
+ '**/*.metal',
21
+ '**/*.comp',
22
+ '**/*.glsl',
23
+ '**/*.wgsl'
24
+ ]
25
+
26
+ pull_request:
27
+ types: [opened, synchronize, reopened]
28
+ paths: [
29
+ '.github/workflows/build-self-hosted.yml',
30
+ '**/CMakeLists.txt',
31
+ '**/.cmake',
32
+ '**/*.h',
33
+ '**/*.hpp',
34
+ '**/*.c',
35
+ '**/*.cpp',
36
+ '**/*.cu',
37
+ '**/*.cuh',
38
+ '**/*.swift',
39
+ '**/*.m',
40
+ '**/*.metal',
41
+ '**/*.comp',
42
+ '**/*.glsl',
43
+ '**/*.wgsl'
44
+ ]
45
+
46
+ concurrency:
47
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
48
+ cancel-in-progress: true
49
+
50
+ env:
51
+ # note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
52
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
53
+ GGML_NLOOP: 3
54
+ GGML_N_THREADS: 1
55
+ LLAMA_ARG_LOG_COLORS: 1
56
+ LLAMA_ARG_LOG_PREFIX: 1
57
+ LLAMA_ARG_LOG_TIMESTAMPS: 1
58
+
59
+ jobs:
60
+ gpu-cuda:
61
+ runs-on: [self-hosted, Linux, NVIDIA]
62
+
63
+ steps:
64
+ - name: Clone
65
+ id: checkout
66
+ uses: actions/checkout@v6
67
+
68
+ - name: Test
69
+ id: ggml-ci
70
+ run: |
71
+ nvidia-smi
72
+ GG_BUILD_CUDA=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
73
+
74
+ gpu-rocm:
75
+ runs-on: [self-hosted, Linux, AMD]
76
+
77
+ steps:
78
+ - name: Clone
79
+ id: checkout
80
+ uses: actions/checkout@v6
81
+
82
+ - name: Test
83
+ id: ggml-ci
84
+ # HIP_LAUNCH_BLOCKING=1: workaround for an async-execution correctness
85
+ # issue on integrated RDNA3.5 (gfx1151) where batched inference returns
86
+ # incorrect output (perplexity ~88 vs ~9.4). Serializing kernel launches
87
+ # restores correctness. Remove once the underlying ROCm/HIP issue is fixed.
88
+ env:
89
+ HIP_LAUNCH_BLOCKING: "1"
90
+ run: |
91
+ rocminfo
92
+ GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS=gfx1151 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
93
+
94
+ gpu-vulkan-nvidia-cm:
95
+ runs-on: [self-hosted, Linux, NVIDIA]
96
+
97
+ steps:
98
+ - name: Clone
99
+ id: checkout
100
+ uses: actions/checkout@v6
101
+
102
+ - name: Test
103
+ id: ggml-ci
104
+ run: |
105
+ vulkaninfo --summary
106
+ GG_BUILD_VULKAN=1 GGML_VK_DISABLE_COOPMAT2=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
107
+
108
+ gpu-vulkan-nvidia-cm2:
109
+ runs-on: [self-hosted, Linux, NVIDIA, COOPMAT2]
110
+
111
+ steps:
112
+ - name: Clone
113
+ id: checkout
114
+ uses: actions/checkout@v6
115
+
116
+ - name: Test
117
+ id: ggml-ci
118
+ run: |
119
+ vulkaninfo --summary
120
+ GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
121
+
122
+ gpu-webgpu-nvidia:
123
+ runs-on: [self-hosted, Linux, NVIDIA, X64]
124
+
125
+ steps:
126
+ - name: Clone
127
+ id: checkout
128
+ uses: actions/checkout@v6
129
+
130
+ - name: Dawn Dependency
131
+ id: dawn-depends
132
+ run: |
133
+ DAWN_VERSION="v20260317.182325"
134
+ DAWN_OWNER="google"
135
+ DAWN_REPO="dawn"
136
+ DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-ubuntu-latest-Release"
137
+ echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
138
+ curl -L -o artifact.tar.gz \
139
+ "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
140
+ mkdir dawn
141
+ tar -xvf artifact.tar.gz -C dawn --strip-components=1
142
+
143
+ - name: Test
144
+ id: ggml-ci
145
+ run: |
146
+ GG_BUILD_WEBGPU=1 \
147
+ GG_BUILD_WEBGPU_DAWN_PREFIX="$GITHUB_WORKSPACE/dawn" \
148
+ GG_BUILD_WEBGPU_DAWN_DIR="$GITHUB_WORKSPACE/dawn/lib64/cmake/Dawn" \
149
+ bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
150
+
151
+ # TODO: provision AMX-compatible machine
152
+ #cpu-amx:
153
+ # runs-on: [self-hosted, Linux, CPU, AMX]
154
+
155
+ # steps:
156
+ # - name: Clone
157
+ # id: checkout
158
+ # uses: actions/checkout@v6
159
+
160
+ # - name: Test
161
+ # id: ggml-ci
162
+ # run: |
163
+ # bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
164
+
165
+ # TODO: provision AMD GPU machine
166
+ # amd-vulkan:
167
+ # runs-on: [self-hosted, Linux, AMD]
168
+
169
+ # steps:
170
+ # - name: Clone
171
+ # id: checkout
172
+ # uses: actions/checkout@v6
173
+
174
+ # - name: Test
175
+ # id: ggml-ci
176
+ # run: |
177
+ # vulkaninfo --summary
178
+ # GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
179
+
180
+ # TODO: provision AMD GPU machine
181
+ # amd-rocm:
182
+ # runs-on: [self-hosted, Linux, AMD]
183
+
184
+ # steps:
185
+ # - name: Clone
186
+ # id: checkout
187
+ # uses: actions/checkout@v6
188
+
189
+ # - name: Test
190
+ # id: ggml-ci
191
+ # run: |
192
+ # amd-smi static
193
+ # GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS="gfx1101" bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
194
+
195
+ gpu-metal:
196
+ runs-on: [self-hosted, macOS, ARM64]
197
+
198
+ steps:
199
+ - name: Clone
200
+ id: checkout
201
+ uses: actions/checkout@v6
202
+
203
+ - name: Test
204
+ id: ggml-ci
205
+ run: |
206
+ GG_BUILD_METAL=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
207
+
208
+ gpu-webgpu-apple:
209
+ runs-on: [self-hosted, macOS, ARM64]
210
+
211
+ steps:
212
+ - name: Clone
213
+ id: checkout
214
+ uses: actions/checkout@v6
215
+
216
+ - name: Dawn Dependency
217
+ id: dawn-depends
218
+ run: |
219
+ DAWN_VERSION="v20260317.182325"
220
+ DAWN_OWNER="google"
221
+ DAWN_REPO="dawn"
222
+ DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-macos-latest-Release"
223
+ echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
224
+ curl -L -o artifact.tar.gz \
225
+ "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
226
+ mkdir dawn
227
+ tar -xvf artifact.tar.gz -C dawn --strip-components=1
228
+
229
+ - name: Test
230
+ id: ggml-ci
231
+ run: |
232
+ GG_BUILD_WEBGPU=1 GG_BUILD_WEBGPU_DAWN_PREFIX="$GITHUB_WORKSPACE/dawn" \
233
+ bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
234
+
235
+ gpu-vulkan-apple:
236
+ runs-on: [self-hosted, macOS, ARM64]
237
+
238
+ steps:
239
+ - name: Clone
240
+ id: checkout
241
+ uses: actions/checkout@v6
242
+
243
+ - name: Test
244
+ id: ggml-ci
245
+ run: |
246
+ vulkaninfo --summary
247
+ GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
248
+
249
+ gpu-vulkan-intel-linux:
250
+ runs-on: [self-hosted, Linux, Intel]
251
+
252
+ steps:
253
+ - name: Clone
254
+ id: checkout
255
+ uses: actions/checkout@v6
256
+ with:
257
+ persist-credentials: false
258
+
259
+ - name: Test
260
+ id: ggml-ci
261
+ run: |
262
+ vulkaninfo --summary
263
+ GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
264
+
265
+ gpu-vulkan-intel-windows:
266
+ runs-on: [self-hosted, Windows, X64, Intel]
267
+
268
+ steps:
269
+ - name: Clone
270
+ id: checkout
271
+ uses: actions/checkout@v6
272
+
273
+ - name: Test
274
+ id: ggml-ci
275
+ shell: C:\msys64\usr\bin\bash.exe --noprofile --norc -eo pipefail "{0}"
276
+ env:
277
+ MSYSTEM: UCRT64
278
+ CHERE_INVOKING: 1
279
+ PATH: C:\msys64\ucrt64\bin;C:\msys64\usr\bin;C:\Windows\System32;${{ env.PATH }}
280
+ run: |
281
+ vulkaninfo --summary
282
+ # Skip python related tests with GG_BUILD_LOW_PERF=1 since Windows MSYS2 UCRT64 currently fails to create
283
+ # a valid python environment for testing
284
+ LLAMA_FATAL_WARNINGS=OFF GG_BUILD_NINJA=1 GG_BUILD_VULKAN=1 GG_BUILD_LOW_PERF=1 ./ci/run.sh ./results/llama.cpp ./mnt/llama.cpp
285
+
286
+ gpu-openvino-low-perf:
287
+ runs-on: [self-hosted, Linux, Intel, OpenVINO]
288
+
289
+ env:
290
+ # Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
291
+ OPENVINO_VERSION_MAJOR: "2026.3.1"
292
+ OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
293
+
294
+ steps:
295
+ - name: Clone
296
+ id: checkout
297
+ uses: actions/checkout@v6
298
+
299
+ - name: Setup OpenVINO Toolkit
300
+ uses: ./.github/actions/linux-setup-openvino
301
+ with:
302
+ path: ./openvino_toolkit
303
+ version_major: ${{ env.OPENVINO_VERSION_MAJOR }}
304
+ version_full: ${{ env.OPENVINO_VERSION_FULL }}
305
+
306
+ - name: Install OpenVINO dependencies
307
+ run: |
308
+ cd ./openvino_toolkit
309
+ chmod +x ./install_dependencies/install_openvino_dependencies.sh
310
+ echo "Y" | sudo -E ./install_dependencies/install_openvino_dependencies.sh
311
+
312
+ - name: Test
313
+ id: ggml-ci
314
+ run: |
315
+ source ./openvino_toolkit/setupvars.sh
316
+ GG_BUILD_OPENVINO=1 GGML_OPENVINO_DEVICE=GPU GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
317
+
318
+ cpu-x64-high-perf:
319
+ runs-on: [self-hosted, Linux, X64]
320
+
321
+ steps:
322
+ - name: Clone
323
+ id: checkout
324
+ uses: actions/checkout@v6
325
+
326
+ - name: Test
327
+ id: ggml-ci
328
+ run: |
329
+ LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
330
+
331
+ cpu-arm64-high-perf-graviton4:
332
+ runs-on: ah-ubuntu_22_04-c8g_8x
333
+
334
+ steps:
335
+ - name: Clone
336
+ id: checkout
337
+ uses: actions/checkout@v6
338
+
339
+ - name: Dependencies
340
+ id: depends
341
+ run: |
342
+ set -euxo pipefail
343
+ sudo apt-get update
344
+ sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a \
345
+ apt-get install -y \
346
+ build-essential \
347
+ python3-venv \
348
+ gpg \
349
+ wget \
350
+ time \
351
+ git-lfs
352
+
353
+ git lfs install
354
+
355
+ # install the latest cmake
356
+ sudo install -d /usr/share/keyrings
357
+ wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc \
358
+ | gpg --dearmor \
359
+ | sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
360
+ echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ jammy main' \
361
+ | sudo tee /etc/apt/sources.list.d/kitware.list
362
+ sudo apt-get update
363
+ sudo apt-get install -y cmake
364
+
365
+ - name: Test
366
+ id: ggml-ci
367
+ run: |
368
+ LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 GG_BUILD_NO_BF16=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
369
+
370
+ cpu-arm64-graviton4-kleidiai:
371
+ runs-on: ah-ubuntu_22_04-c8g_8x
372
+
373
+ steps:
374
+ - name: Clone
375
+ id: checkout
376
+ uses: actions/checkout@v6
377
+
378
+ - name: Dependencies
379
+ id: depends
380
+ run: |
381
+ set -euxo pipefail
382
+ sudo apt-get update
383
+ sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a \
384
+ apt-get install -y \
385
+ build-essential \
386
+ python3-venv \
387
+ gpg \
388
+ wget \
389
+ time \
390
+ git-lfs
391
+
392
+ git lfs install
393
+
394
+ # install the latest cmake
395
+ sudo install -d /usr/share/keyrings
396
+ wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc \
397
+ | gpg --dearmor \
398
+ | sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
399
+ echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ jammy main' \
400
+ | sudo tee /etc/apt/sources.list.d/kitware.list
401
+ sudo apt-get update
402
+ sudo apt-get install -y cmake
403
+
404
+ - name: Test
405
+ id: ggml-ci
406
+ run: |
407
+ GG_BUILD_KLEIDIAI=1 \
408
+ GG_BUILD_EXTRA_TESTS_0=1 \
409
+ bash ./ci/run.sh ./tmp/results ./tmp/mnt
.github/workflows/build-sycl.yml ADDED
@@ -0,0 +1,171 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: CI (sycl)
2
+
3
+ on:
4
+ workflow_dispatch: # allows manual triggering
5
+ push:
6
+ branches:
7
+ - master
8
+ paths: [
9
+ '.github/workflows/build-sycl.yml',
10
+ '**/CMakeLists.txt',
11
+ '**/.cmake',
12
+ '**/*.h',
13
+ '**/*.hpp',
14
+ '**/*.c',
15
+ '**/*.cpp'
16
+ ]
17
+
18
+ pull_request:
19
+ types: [opened, synchronize, reopened]
20
+ paths: [
21
+ '.github/workflows/build-sycl.yml',
22
+ 'ggml/src/ggml-sycl/**'
23
+ ]
24
+
25
+ concurrency:
26
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
27
+ cancel-in-progress: true
28
+
29
+ env:
30
+ GGML_NLOOP: 3
31
+ GGML_N_THREADS: 1
32
+ LLAMA_ARG_LOG_COLORS: 1
33
+ LLAMA_ARG_LOG_PREFIX: 1
34
+ LLAMA_ARG_LOG_TIMESTAMPS: 1
35
+
36
+ jobs:
37
+ ubuntu-24-sycl:
38
+ strategy:
39
+ matrix:
40
+ build: [fp32, fp16]
41
+ include:
42
+ - build: fp32
43
+ fp16: OFF
44
+ - build: fp16
45
+ fp16: ON
46
+
47
+ runs-on: ubuntu-24.04
48
+
49
+ env:
50
+ ONEAPI_ROOT: /opt/intel/oneapi/
51
+ ONEAPI_INSTALLER_VERSION: "2025.3.3"
52
+ LEVEL_ZERO_VERSION: "1.28.2"
53
+ LEVEL_ZERO_UBUNTU_VERSION: "u24.04"
54
+
55
+ continue-on-error: true
56
+
57
+ steps:
58
+ - name: Clone
59
+ id: checkout
60
+ uses: actions/checkout@v6
61
+
62
+ - name: Download & Install oneAPI
63
+ shell: bash
64
+ run: |
65
+ cd /tmp
66
+ wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/56f7923a-adb8-43f3-8b02-2b60fcac8cab/intel-deep-learning-essentials-2025.3.3.16_offline.sh -O intel-deep-learning-essentials_offline.sh
67
+ sudo bash intel-deep-learning-essentials_offline.sh -s -a --silent --eula accept
68
+
69
+ - name: Install Level Zero SDK
70
+ shell: bash
71
+ run: |
72
+ cd /tmp
73
+ wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero.deb
74
+ wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero-devel_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero-devel.deb
75
+ sudo apt-get install -y ./level-zero.deb ./level-zero-devel.deb
76
+
77
+ - name: ccache
78
+ uses: ggml-org/ccache-action@v1.2.24
79
+ with:
80
+ key: sycl-ubuntu-24-${{ matrix.build }}
81
+ save: false
82
+
83
+ - name: ccache-buckets-restore
84
+ uses: ./.github/actions/ccache-buckets
85
+ env:
86
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
87
+ with:
88
+ key: sycl-ubuntu-24-${{ matrix.build }}
89
+ folder: llama.cpp
90
+ hf_bucket: ggml-org/cache
91
+
92
+ - name: Build
93
+ id: cmake_build
94
+ run: |
95
+ source /opt/intel/oneapi/setvars.sh
96
+ cmake -B build \
97
+ -G "Ninja" \
98
+ -DCMAKE_BUILD_TYPE=Release \
99
+ -DGGML_SYCL=ON \
100
+ -DCMAKE_C_COMPILER=icx \
101
+ -DCMAKE_CXX_COMPILER=icpx \
102
+ -DLLAMA_OPENSSL=OFF \
103
+ -DGGML_NATIVE=OFF \
104
+ -DGGML_SYCL_F16=${{ matrix.fp16 }}
105
+ time cmake --build build --config Release -j $(nproc)
106
+
107
+ - name: ccache-buckets-save
108
+ if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
109
+ uses: ./.github/actions/ccache-buckets
110
+ env:
111
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
112
+ with:
113
+ key: sycl-ubuntu-24-${{ matrix.build }}
114
+ folder: llama.cpp
115
+ evict-old-files: 1d
116
+ hf_bucket: ggml-org/cache
117
+ save: true
118
+
119
+ windows-latest-sycl:
120
+ runs-on: windows-2022
121
+
122
+ defaults:
123
+ run:
124
+ shell: bash
125
+
126
+ env:
127
+ WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b60765d1-2b85-4e85-86b6-cb0e9563a699/intel-deep-learning-essentials-2025.3.3.18_offline.exe
128
+ WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.dnnl:intel.oneapi.win.tbb.devel
129
+ LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.28.2/level-zero-win-sdk-1.28.2.zip
130
+ ONEAPI_ROOT: "C:/Program Files (x86)/Intel/oneAPI"
131
+ ONEAPI_INSTALLER_VERSION: "2025.3.3"
132
+ steps:
133
+ - name: Clone
134
+ id: checkout
135
+ uses: actions/checkout@v6
136
+
137
+ - name: Download & Install oneAPI
138
+ shell: bash
139
+ run: |
140
+ scripts/install-oneapi.bat $WINDOWS_BASEKIT_URL $WINDOWS_DPCPP_MKL
141
+
142
+ - name: Install Level Zero SDK
143
+ shell: pwsh
144
+ run: |
145
+ Invoke-WebRequest -Uri "${{ env.LEVEL_ZERO_SDK_URL }}" -OutFile "level-zero-win-sdk.zip"
146
+ Expand-Archive -Path "level-zero-win-sdk.zip" -DestinationPath "C:/level-zero-sdk" -Force
147
+ "LEVEL_ZERO_V1_SDK_PATH=C:/level-zero-sdk" | Out-File -FilePath $env:GITHUB_ENV -Append
148
+
149
+ - name: ccache
150
+ uses: ggml-org/ccache-action@v1.2.24
151
+ with:
152
+ key: sycl-windows-latest
153
+ variant: ccache
154
+ evict-old-files: 1d
155
+ save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
156
+
157
+ # TODO: add ssl support ; we will also need to modify win-build-sycl.bat to accept user-specified args
158
+
159
+ - name: Build
160
+ id: cmake_build
161
+ run: examples/sycl/win-build-sycl.bat
162
+
163
+ - name: ccache-clear
164
+ uses: ./.github/actions/ccache-clear
165
+ env:
166
+ GH_TOKEN: ${{ github.token }}
167
+ with:
168
+ key: sycl-windows-latest
169
+ older: 5m
170
+ min: 1
171
+ dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
.github/workflows/build-virtgpu.yml ADDED
@@ -0,0 +1,50 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: CI (virtgpu)
2
+
3
+ on:
4
+ workflow_dispatch: # allows manual triggering
5
+ push:
6
+ branches:
7
+ - master
8
+ paths: [
9
+ '.github/workflows/build-virtgpu.yml',
10
+ '**/CMakeLists.txt',
11
+ '**/.cmake',
12
+ '**/*.h',
13
+ '**/*.hpp',
14
+ '**/*.c',
15
+ '**/*.cpp'
16
+ ]
17
+
18
+ pull_request:
19
+ types: [opened, synchronize, reopened]
20
+ paths: [
21
+ '.github/workflows/build-virtgpu.yml',
22
+ 'ggml/src/ggml-virtgpu/**'
23
+ ]
24
+
25
+ concurrency:
26
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
27
+ cancel-in-progress: true
28
+
29
+ jobs:
30
+ ubuntu-24-virtgpu:
31
+ runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
32
+
33
+ steps:
34
+ - name: Clone
35
+ id: checkout
36
+ uses: actions/checkout@v6
37
+
38
+ - name: Dependencies
39
+ id: depends
40
+ run: |
41
+ sudo apt-get update
42
+ sudo apt-get install -y build-essential libdrm-dev pkg-config libssl-dev
43
+
44
+ - name: Build
45
+ id: cmake_build
46
+ run: |
47
+ cmake -B build \
48
+ -DGGML_VIRTGPU=ON \
49
+ -DGGML_VIRTGPU_BACKEND=ON
50
+ cmake --build build --config Release -j $(nproc)
.github/workflows/build-vulkan.yml ADDED
@@ -0,0 +1,232 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: CI (vulkan)
2
+
3
+ on:
4
+ workflow_dispatch: # allows manual triggering
5
+ push:
6
+ branches:
7
+ - master
8
+ paths: [
9
+ '.github/workflows/build-vulkan.yml',
10
+ '**/CMakeLists.txt',
11
+ '**/.cmake',
12
+ '**/*.h',
13
+ '**/*.hpp',
14
+ '**/*.c',
15
+ '**/*.cpp',
16
+ '**/*.comp',
17
+ '**/*.glsl'
18
+ ]
19
+
20
+ pull_request:
21
+ types: [opened, synchronize, reopened]
22
+ paths: [
23
+ '.github/workflows/build-vulkan.yml',
24
+ 'ggml/src/ggml-vulkan/**'
25
+ ]
26
+
27
+ concurrency:
28
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
29
+ cancel-in-progress: true
30
+
31
+ env:
32
+ GGML_NLOOP: 3
33
+ GGML_N_THREADS: 1
34
+ LLAMA_ARG_LOG_COLORS: 1
35
+ LLAMA_ARG_LOG_PREFIX: 1
36
+ LLAMA_ARG_LOG_TIMESTAMPS: 1
37
+
38
+ jobs:
39
+ ubuntu-arm64:
40
+ runs-on: ubuntu-24.04-arm
41
+
42
+ steps:
43
+ - name: Clone
44
+ id: checkout
45
+ uses: actions/checkout@v6
46
+
47
+ - name: Dependencies
48
+ id: depends
49
+ run: |
50
+ sudo apt-get update
51
+ sudo apt-get install -y gcc-14 g++-14 build-essential glslc libvulkan-dev spirv-headers libssl-dev ninja-build
52
+ echo "CC=gcc-14" >> "$GITHUB_ENV"
53
+ echo "CXX=g++-14" >> "$GITHUB_ENV"
54
+
55
+ - name: ccache
56
+ uses: ggml-org/ccache-action@v1.2.24
57
+ with:
58
+ key: vulkan-ubuntu-24.04-arm
59
+ variant: ccache
60
+ save: false
61
+
62
+ - name: ccache-buckets-restore
63
+ uses: ./.github/actions/ccache-buckets
64
+ env:
65
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
66
+ with:
67
+ key: vulkan-ubuntu-24.04-arm
68
+ folder: llama.cpp
69
+ hf_bucket: ggml-org/cache
70
+
71
+ - name: Configure
72
+ id: cmake_configure
73
+ run: |
74
+ cmake -B build \
75
+ -G "Ninja" \
76
+ -DCMAKE_BUILD_TYPE=Release \
77
+ -DGGML_VULKAN=ON
78
+
79
+ - name: Build
80
+ id: cmake_build
81
+ run: |
82
+ time cmake --build build -j $(nproc)
83
+
84
+ - name: ccache-buckets-save
85
+ if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
86
+ uses: ./.github/actions/ccache-buckets
87
+ env:
88
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
89
+ with:
90
+ key: vulkan-ubuntu-24.04-arm
91
+ folder: llama.cpp
92
+ evict-old-files: 1d
93
+ hf_bucket: ggml-org/cache
94
+ save: true
95
+
96
+ ubuntu-llvmpipe:
97
+ runs-on: ubuntu-24.04
98
+
99
+ steps:
100
+ - name: Clone
101
+ id: checkout
102
+ uses: actions/checkout@v6
103
+
104
+ - name: Dependencies
105
+ id: depends
106
+ run: |
107
+ sudo add-apt-repository -y ppa:kisak/kisak-mesa
108
+ sudo apt-get update -y
109
+ sudo apt-get install -y build-essential mesa-vulkan-drivers libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libssl-dev
110
+
111
+ - name: Get latest Vulkan SDK version
112
+ id: vulkan_sdk_version
113
+ run: |
114
+ echo "VULKAN_SDK_VERSION=$(curl https://vulkan.lunarg.com/sdk/latest/linux.txt)" >> "$GITHUB_ENV"
115
+
116
+ - name: Setup Vulkan SDK
117
+ id: setup
118
+ uses: ./.github/actions/unarchive-tar
119
+ with:
120
+ url: https://sdk.lunarg.com/sdk/download/${{ env.VULKAN_SDK_VERSION }}/linux/vulkan_sdk.tar.xz
121
+ path: ./vulkan_sdk
122
+ strip: 1
123
+
124
+ - name: ccache
125
+ uses: ggml-org/ccache-action@v1.2.24
126
+ with:
127
+ key: vulkan-ubuntu-24.04-llvmpipe
128
+ save: false
129
+
130
+ - name: ccache-buckets-restore
131
+ uses: ./.github/actions/ccache-buckets
132
+ env:
133
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
134
+ with:
135
+ key: vulkan-ubuntu-24.04-llvmpipe
136
+ folder: llama.cpp
137
+ hf_bucket: ggml-org/cache
138
+
139
+ - name: Build
140
+ id: cmake_build
141
+ run: |
142
+ source ./vulkan_sdk/setup-env.sh
143
+ cmake -B build \
144
+ -DGGML_NATIVE=OFF \
145
+ -DGGML_VULKAN=ON
146
+ cmake --build build --config Release -j $(nproc)
147
+
148
+ - name: ccache-buckets-save
149
+ if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
150
+ uses: ./.github/actions/ccache-buckets
151
+ env:
152
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
153
+ with:
154
+ key: vulkan-ubuntu-24.04-llvmpipe
155
+ folder: llama.cpp
156
+ evict-old-files: 1d
157
+ hf_bucket: ggml-org/cache
158
+ save: true
159
+
160
+ - name: Test
161
+ id: cmake_test
162
+ run: |
163
+ cd build
164
+ export GGML_VK_VISIBLE_DEVICES=0
165
+ export GGML_VK_DISABLE_F16=1
166
+ export GGML_VK_DISABLE_COOPMAT=1
167
+ # This is using llvmpipe and runs slower than other backends
168
+ # test-backend-ops is too slow on llvmpipe, skip it
169
+ ctest -L main -E test-backend-ops --verbose --timeout 900
170
+
171
+ windows:
172
+ runs-on: windows-2025
173
+
174
+ env:
175
+ VULKAN_VERSION: 1.4.357.0
176
+
177
+ steps:
178
+ - name: Clone
179
+ id: checkout
180
+ uses: actions/checkout@v6
181
+
182
+ - name: ccache
183
+ uses: ggml-org/ccache-action@v1.2.24
184
+ with:
185
+ key: cpu-windows-2025-x64-vulkan
186
+ variant: ccache
187
+ evict-old-files: 1d
188
+ save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
189
+
190
+ - name: Install Vulkan SDK
191
+ id: get_vulkan
192
+ run: |
193
+ curl.exe -o $env:RUNNER_TEMP/VulkanSDK-Installer.exe -L "https://sdk.lunarg.com/sdk/download/${env:VULKAN_VERSION}/windows/vulkansdk-windows-X64-${env:VULKAN_VERSION}.exe"
194
+ & "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install
195
+ Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\${env:VULKAN_VERSION}"
196
+ Add-Content $env:GITHUB_PATH "C:\VulkanSDK\${env:VULKAN_VERSION}\bin"
197
+
198
+ - name: Install Ninja
199
+ id: install_ninja
200
+ run: |
201
+ choco install ninja
202
+
203
+ - name: Build
204
+ id: cmake_build
205
+ run: |
206
+ cmake -S . -B build -G "Ninja Multi-Config" `
207
+ -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake `
208
+ -DCMAKE_BUILD_TYPE=Release `
209
+ -DGGML_NATIVE=OFF `
210
+ -DLLAMA_BUILD_SERVER=ON `
211
+ -DGGML_RPC=ON `
212
+ -DGGML_BACKEND_DL=ON `
213
+ -DGGML_CPU_ALL_VARIANTS=ON `
214
+ -DGGML_VULKAN=ON `
215
+ -DLLAMA_BUILD_BORINGSSL=ON
216
+ cmake --build build --config Release -j ${env:NUMBER_OF_PROCESSORS}
217
+
218
+ - name: Test
219
+ id: cmake_test
220
+ run: |
221
+ cd build
222
+ ctest -L main -C Release --verbose --timeout 900
223
+
224
+ - name: ccache-clear
225
+ uses: ./.github/actions/ccache-clear
226
+ env:
227
+ GH_TOKEN: ${{ github.token }}
228
+ with:
229
+ key: cpu-windows-2025-x64-vulkan
230
+ older: 5m
231
+ min: 1
232
+ dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
.github/workflows/build-wasm.yml ADDED
@@ -0,0 +1,110 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: CI (wasm)
2
+
3
+ on:
4
+ workflow_dispatch: # allows manual triggering
5
+ push:
6
+ branches:
7
+ - master
8
+ paths: [
9
+ '.github/workflows/build-wasm.yml',
10
+ '**/CMakeLists.txt',
11
+ '**/.cmake',
12
+ '**/*.h',
13
+ '**/*.hpp',
14
+ '**/*.c',
15
+ '**/*.cpp',
16
+ '**/*.wgsl',
17
+ '**/*.tmpl',
18
+ 'ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py'
19
+ ]
20
+
21
+ pull_request:
22
+ types: [opened, synchronize, reopened]
23
+ paths: [
24
+ '.github/workflows/build-wasm.yml',
25
+ '**/CMakeLists.txt',
26
+ '**/.cmake',
27
+ '**/*.h',
28
+ '**/*.hpp',
29
+ '**/*.c',
30
+ '**/*.cpp',
31
+ '**/*.wgsl',
32
+ '**/*.tmpl',
33
+ 'ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py'
34
+ ]
35
+
36
+ concurrency:
37
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
38
+ cancel-in-progress: true
39
+
40
+ env:
41
+ GGML_NLOOP: 3
42
+ GGML_N_THREADS: 1
43
+ LLAMA_ARG_LOG_COLORS: 1
44
+ LLAMA_ARG_LOG_PREFIX: 1
45
+ LLAMA_ARG_LOG_TIMESTAMPS: 1
46
+
47
+ jobs:
48
+ ubuntu-webgpu:
49
+ runs-on: ubuntu-24.04-arm
50
+
51
+ steps:
52
+ - name: Clone
53
+ id: checkout
54
+ uses: actions/checkout@v6
55
+
56
+ - name: ccache
57
+ uses: ggml-org/ccache-action@v1.2.24
58
+ with:
59
+ key: webgpu-ubuntu-24.04-arm-wasm
60
+ save: false
61
+
62
+ - name: Install Emscripten
63
+ run: |
64
+ git clone https://github.com/emscripten-core/emsdk.git
65
+ cd emsdk
66
+ ./emsdk install latest
67
+ ./emsdk activate latest
68
+
69
+ - name: Fetch emdawnwebgpu
70
+ run: |
71
+ DAWN_TAG="v20260317.182325"
72
+ EMDAWN_PKG="emdawnwebgpu_pkg-${DAWN_TAG}.zip"
73
+ echo "Downloading ${EMDAWN_PKG}"
74
+ curl -L -o emdawn.zip \
75
+ "https://github.com/google/dawn/releases/download/${DAWN_TAG}/${EMDAWN_PKG}"
76
+ unzip emdawn.zip
77
+
78
+ - name: ccache-buckets-restore
79
+ uses: ./.github/actions/ccache-buckets
80
+ env:
81
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
82
+ with:
83
+ key: webgpu-ubuntu-24.04-arm-wasm
84
+ folder: llama.cpp
85
+ hf_bucket: ggml-org/cache
86
+
87
+ - name: Build WASM WebGPU
88
+ run: |
89
+ source emsdk/emsdk_env.sh
90
+ emcmake cmake -B build-wasm \
91
+ -G "Ninja" \
92
+ -DCMAKE_BUILD_TYPE=Release \
93
+ -DGGML_WEBGPU=ON \
94
+ -DGGML_OPENMP=OFF \
95
+ -DLLAMA_OPENSSL=OFF \
96
+ -DEMDAWNWEBGPU_DIR=emdawnwebgpu_pkg
97
+
98
+ time cmake --build build-wasm --config Release --target test-backend-ops -j $(nproc)
99
+
100
+ - name: ccache-buckets-save
101
+ if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
102
+ uses: ./.github/actions/ccache-buckets
103
+ env:
104
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
105
+ with:
106
+ key: webgpu-ubuntu-24.04-arm-wasm
107
+ folder: llama.cpp
108
+ evict-old-files: 1d
109
+ hf_bucket: ggml-org/cache
110
+ save: true
.github/workflows/build-webgpu.yml ADDED
@@ -0,0 +1,195 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ name: CI (webgpu)
2
+
3
+ on:
4
+ workflow_dispatch: # allows manual triggering
5
+ push:
6
+ branches:
7
+ - master
8
+ paths: [
9
+ '.github/workflows/build-webgpu.yml',
10
+ '**/CMakeLists.txt',
11
+ '**/.cmake',
12
+ '**/*.h',
13
+ '**/*.hpp',
14
+ '**/*.c',
15
+ '**/*.cpp',
16
+ '**/*.wgsl',
17
+ '**/*.tmpl',
18
+ 'ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py'
19
+ ]
20
+
21
+ pull_request:
22
+ types: [opened, synchronize, reopened]
23
+ paths: [
24
+ '.github/workflows/build-webgpu.yml',
25
+ 'ggml/src/ggml-webgpu/**'
26
+ ]
27
+
28
+ concurrency:
29
+ group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
30
+ cancel-in-progress: true
31
+
32
+ env:
33
+ GGML_NLOOP: 3
34
+ GGML_N_THREADS: 1
35
+ LLAMA_ARG_LOG_COLORS: 1
36
+ LLAMA_ARG_LOG_PREFIX: 1
37
+ LLAMA_ARG_LOG_TIMESTAMPS: 1
38
+
39
+ jobs:
40
+ format:
41
+ runs-on: ubuntu-24.04
42
+
43
+ steps:
44
+ - name: Clone
45
+ uses: actions/checkout@v6
46
+
47
+ - name: Install clang-format 22
48
+ run: |
49
+ wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key |
50
+ sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc > /dev/null
51
+ sudo add-apt-repository -y \
52
+ "deb http://apt.llvm.org/noble/ llvm-toolchain-noble-22 main"
53
+ sudo apt-get update
54
+ sudo apt-get install -y clang-format-22
55
+
56
+ - name: Check formatting
57
+ run: |
58
+ find ggml/src/ggml-webgpu \
59
+ -type f \( -name '*.cpp' -o -name '*.hpp' -o -name '*.h' \) \
60
+ -print0 |
61
+ xargs -0 clang-format-22 --dry-run --Werror
62
+
63
+ macos:
64
+ runs-on: macos-latest
65
+
66
+ steps:
67
+ - name: Clone
68
+ id: checkout
69
+ uses: actions/checkout@v6
70
+
71
+ - name: ccache
72
+ uses: ggml-org/ccache-action@v1.2.24
73
+ with:
74
+ key: webgpu-macos-latest
75
+ save: false
76
+
77
+ - name: Dawn Dependency
78
+ id: dawn-depends
79
+ run: |
80
+ DAWN_VERSION="v20260317.182325"
81
+ DAWN_OWNER="google"
82
+ DAWN_REPO="dawn"
83
+ DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-macos-latest-Release"
84
+ echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
85
+ curl -L -o artifact.tar.gz \
86
+ "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
87
+ mkdir dawn
88
+ tar -xvf artifact.tar.gz -C dawn --strip-components=1
89
+
90
+ - name: ccache-buckets-restore
91
+ uses: ./.github/actions/ccache-buckets
92
+ env:
93
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
94
+ with:
95
+ key: webgpu-macos-latest
96
+ folder: llama.cpp
97
+ hf_bucket: ggml-org/cache
98
+
99
+ - name: Build
100
+ id: cmake_build
101
+ run: |
102
+ export CMAKE_PREFIX_PATH=dawn
103
+ cmake -B build -G "Ninja" -DCMAKE_BUILD_TYPE=Release -DGGML_WEBGPU=ON -DGGML_METAL=OFF -DGGML_BLAS=OFF
104
+ time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
105
+
106
+ - name: ccache-buckets-save
107
+ if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
108
+ uses: ./.github/actions/ccache-buckets
109
+ env:
110
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
111
+ with:
112
+ key: webgpu-macos-latest
113
+ folder: llama.cpp
114
+ evict-old-files: 1d
115
+ hf_bucket: ggml-org/cache
116
+ save: true
117
+
118
+ - name: Test
119
+ id: cmake_test
120
+ run: |
121
+ cd build
122
+ ctest -L main --verbose --timeout 900
123
+
124
+ ubuntu:
125
+ runs-on: ubuntu-24.04
126
+
127
+ steps:
128
+ - name: Clone
129
+ id: checkout
130
+ uses: actions/checkout@v6
131
+
132
+ - name: ccache
133
+ uses: ggml-org/ccache-action@v1.2.24
134
+ with:
135
+ key: webgpu-ubuntu-24.04
136
+ save: false
137
+
138
+ - name: Dependencies
139
+ id: depends
140
+ run: |
141
+ sudo add-apt-repository -y ppa:kisak/kisak-mesa
142
+ sudo apt-get update -y
143
+ sudo apt-get install -y build-essential mesa-vulkan-drivers \
144
+ libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libssl-dev
145
+
146
+ - name: Dawn Dependency
147
+ id: dawn-depends
148
+ run: |
149
+ sudo apt-get install -y libxrandr-dev libxinerama-dev libxcursor-dev mesa-common-dev libx11-xcb-dev libxi-dev
150
+ DAWN_VERSION="v20260317.182325"
151
+ DAWN_OWNER="google"
152
+ DAWN_REPO="dawn"
153
+ DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-ubuntu-latest-Release"
154
+ echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
155
+ curl -L -o artifact.tar.gz \
156
+ "https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
157
+ mkdir dawn
158
+ tar -xvf artifact.tar.gz -C dawn --strip-components=1
159
+
160
+ - name: ccache-buckets-restore
161
+ uses: ./.github/actions/ccache-buckets
162
+ env:
163
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
164
+ with:
165
+ key: webgpu-ubuntu-24.04
166
+ folder: llama.cpp
167
+ hf_bucket: ggml-org/cache
168
+
169
+ - name: Build
170
+ id: cmake_build
171
+ run: |
172
+ export Dawn_DIR=dawn/lib64/cmake/Dawn
173
+ cmake -B build \
174
+ -DGGML_WEBGPU=ON
175
+ time cmake --build build --config Release -j $(nproc)
176
+
177
+ - name: ccache-buckets-save
178
+ if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
179
+ uses: ./.github/actions/ccache-buckets
180
+ env:
181
+ HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
182
+ with:
183
+ key: webgpu-ubuntu-24.04
184
+ folder: llama.cpp
185
+ evict-old-files: 1d
186
+ hf_bucket: ggml-org/cache
187
+ save: true
188
+
189
+ - name: Test
190
+ id: cmake_test
191
+ run: |
192
+ cd build
193
+ # This is using llvmpipe and runs slower than other backends
194
+ # test-backend-ops is too slow on llvmpipe, skip it
195
+ ctest -L main -E test-backend-ops --verbose --timeout 900
cmake/build-info.cmake ADDED
@@ -0,0 +1,48 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ set(BUILD_NUMBER 0)
2
+ set(BUILD_COMMIT "unknown")
3
+ set(BUILD_COMPILER "unknown")
4
+ set(BUILD_TARGET "unknown")
5
+
6
+ # Look for git
7
+ find_package(Git)
8
+ if(NOT Git_FOUND)
9
+ find_program(GIT_EXECUTABLE NAMES git git.exe)
10
+ if(GIT_EXECUTABLE)
11
+ set(Git_FOUND TRUE)
12
+ message(STATUS "Found Git: ${GIT_EXECUTABLE}")
13
+ else()
14
+ message(WARNING "Git not found. Build info will not be accurate.")
15
+ endif()
16
+ endif()
17
+
18
+ # Get the commit count and hash
19
+ if(Git_FOUND)
20
+ execute_process(
21
+ COMMAND ${GIT_EXECUTABLE} rev-parse --short HEAD
22
+ WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
23
+ OUTPUT_VARIABLE HEAD
24
+ OUTPUT_STRIP_TRAILING_WHITESPACE
25
+ RESULT_VARIABLE RES
26
+ )
27
+ if (RES EQUAL 0)
28
+ set(BUILD_COMMIT ${HEAD})
29
+ endif()
30
+ execute_process(
31
+ COMMAND ${GIT_EXECUTABLE} rev-list --count HEAD
32
+ WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
33
+ OUTPUT_VARIABLE COUNT
34
+ OUTPUT_STRIP_TRAILING_WHITESPACE
35
+ RESULT_VARIABLE RES
36
+ )
37
+ if (RES EQUAL 0)
38
+ set(BUILD_NUMBER ${COUNT})
39
+ endif()
40
+ endif()
41
+
42
+ set(BUILD_COMPILER "${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION}")
43
+
44
+ if(CMAKE_VS_PLATFORM_NAME)
45
+ set(BUILD_TARGET ${CMAKE_VS_PLATFORM_NAME})
46
+ else()
47
+ set(BUILD_TARGET "${CMAKE_SYSTEM_NAME} ${CMAKE_SYSTEM_PROCESSOR}")
48
+ endif()
common/build-info.cpp.in ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #include "build-info.h"
2
+
3
+ #include <cstdio>
4
+ #include <string>
5
+
6
+ int LLAMA_BUILD_NUMBER = @LLAMA_BUILD_NUMBER@;
7
+ char const * LLAMA_COMMIT = "@LLAMA_BUILD_COMMIT@";
8
+ char const * LLAMA_COMPILER = "@BUILD_COMPILER@";
9
+ char const * LLAMA_BUILD_TARGET = "@BUILD_TARGET@";
10
+
11
+ int llama_build_number(void) {
12
+ return LLAMA_BUILD_NUMBER;
13
+ }
14
+
15
+ const char * llama_commit(void) {
16
+ return LLAMA_COMMIT;
17
+ }
18
+
19
+ const char * llama_compiler(void) {
20
+ return LLAMA_COMPILER;
21
+ }
22
+
23
+ const char * llama_build_target(void) {
24
+ return LLAMA_BUILD_TARGET;
25
+ }
26
+
27
+ const char * llama_build_info(void) {
28
+ static std::string s = "b" + std::to_string(LLAMA_BUILD_NUMBER) + "-" + LLAMA_COMMIT;
29
+ return s.c_str();
30
+ }
31
+
32
+ void llama_print_build_info(const char * llama_version, FILE * stream) {
33
+ fprintf(stream, "version: %s (build %d, commit %s)\n", llama_version, llama_build_number(), llama_commit());
34
+ fprintf(stream, "built with %s for %s\n", llama_compiler(), llama_build_target());
35
+ }
common/build-info.h ADDED
@@ -0,0 +1,13 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #pragma once
2
+
3
+ #include <cstdio>
4
+
5
+ int llama_build_number(void);
6
+
7
+ const char * llama_commit(void);
8
+ const char * llama_compiler(void);
9
+
10
+ const char * llama_build_target(void);
11
+ const char * llama_build_info(void);
12
+
13
+ void llama_print_build_info(const char *, FILE * = stderr);
docs/build-riscv64-spacemit.md ADDED
@@ -0,0 +1,110 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ > [!IMPORTANT]
2
+ > This build documentation is specific only to RISC-V SpacemiT SOCs.
3
+
4
+ ## Build llama.cpp locally (for riscv64)
5
+
6
+ 1. Prepare Toolchain For RISCV
7
+ ~~~
8
+ wget https://github.com/spacemit-com/toolchain/releases/download/v1.2.4/spacemit-toolchain-linux-glibc-x86_64-v1.2.4.tar.xz
9
+ ~~~
10
+
11
+ 2. Build
12
+ Below is the build script: it requires utilizing RISC-V vector instructions for acceleration. Ensure the `GGML_CPU_RISCV64_SPACEMIT` compilation option is enabled. The currently supported optimization version is `RISCV64_SPACEMIT_IME1` and `RISCV64_SPACEMIT_IME2`, corresponding to the `RISCV64_SPACEMIT_IME_SPEC` compilation option. Compiler configurations are defined in the `riscv64-spacemit-linux-gnu-gcc.cmake` file. Please ensure you have installed the RISC-V compiler and set the environment variable via `export RISCV_ROOT_PATH={your_compiler_path}`.
13
+ ```bash
14
+
15
+ cmake -B build \
16
+ -DCMAKE_BUILD_TYPE=Release \
17
+ -DGGML_CPU_RISCV64_SPACEMIT=ON \
18
+ -DGGML_CPU_REPACK=OFF \
19
+ -DLLAMA_OPENSSL=OFF \
20
+ -DGGML_RVV=ON \
21
+ -DGGML_RV_ZVFH=ON \
22
+ -DGGML_RV_ZFH=ON \
23
+ -DGGML_RV_ZICBOP=ON \
24
+ -DGGML_RV_ZIHINTPAUSE=ON \
25
+ -DGGML_RV_ZBA=ON \
26
+ -DCMAKE_TOOLCHAIN_FILE=${PWD}/cmake/riscv64-spacemit-linux-gnu-gcc.cmake \
27
+ -DCMAKE_INSTALL_PREFIX=build/installed
28
+
29
+ cmake --build build --parallel $(nproc) --config Release
30
+
31
+ pushd build
32
+ make install
33
+ popd
34
+ ```
35
+
36
+ ## Simulation
37
+ You can use QEMU to perform emulation on non-RISC-V architectures.
38
+
39
+ 1. Download QEMU
40
+ ~~~
41
+ wget https://archive.spacemit.com/spacemit-ai/qemu/jdsk-qemu-v0.0.14.tar.gz
42
+ ~~~
43
+
44
+ 2. Run Simulation
45
+ After build your llama.cpp, you can run the executable file via QEMU for simulation, for example:
46
+ ~~~
47
+ export QEMU_ROOT_PATH={your QEMU file path}
48
+ export RISCV_ROOT_PATH_IME1={your RISC-V compiler path}
49
+
50
+ ${QEMU_ROOT_PATH}/bin/qemu-riscv64 -L ${RISCV_ROOT_PATH_IME1}/sysroot -cpu max,vlen=256,elen=64,vext_spec=v1.0 ${PWD}/build/bin/llama-cli -m ${PWD}/models/Qwen2.5-0.5B-Instruct-Q4_0.gguf -t 1
51
+ ~~~
52
+
53
+ ## Quantization Support For Matrix
54
+
55
+ | Quantization Type | X60 | A100 |
56
+ | ---: | ---: | ---: |
57
+ | Q2_K | | :heavy_check_mark: |
58
+ | Q3_K | | :heavy_check_mark: |
59
+ | Q4_0 | :heavy_check_mark: | :heavy_check_mark: |
60
+ | Q4_1 | :heavy_check_mark: | :heavy_check_mark: |
61
+ | Q4_K | :heavy_check_mark: | :heavy_check_mark: |
62
+ | Q5_0 | | :heavy_check_mark: |
63
+ | Q5_1 | | :heavy_check_mark: |
64
+ | Q5_K | | :heavy_check_mark: |
65
+ | Q6_K | | :heavy_check_mark: |
66
+ | Q8_0 | | :heavy_check_mark: |
67
+
68
+
69
+ ## Performance
70
+ * Spacemit(R) X60
71
+ ~~~
72
+ model name : Spacemit(R) X60
73
+ isa : rv64imafdcv_zicbom_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zfhmin_zca_zcd_zba_zbb_zbc_zbs_zkt_zve32f_zve32x_zve64d_zve64f_zve64x_zvfh_zvfhmin_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt
74
+ mmu : sv39
75
+ uarch : spacemit,x60
76
+ mvendorid : 0x710
77
+ marchid : 0x8000000058000001
78
+ ~~~
79
+
80
+ | model | size | params | backend | threads | n_ubatch | fa | mmap | test | t/s |
81
+ | ------------------------------ | ---------: | ---------: | ---------- | ------: | -------: | -: | ---: | --------------: | -------------------: |
82
+ | qwen35 2B Q4_1 | 1.19 GiB | 1.88 B | CPU | 4 | 128 | 1 | 0 | pp128 | 10.32 ± 0.02 |
83
+ | qwen35 2B Q4_1 | 1.19 GiB | 1.88 B | CPU | 4 | 128 | 1 | 0 | tg128 | 3.07 ± 0.01 |
84
+ | qwen3 0.6B Q4_0 | 358.78 MiB | 596.05 M | CPU | 4 | 128 | 1 | 0 | pp128 | 49.15 ± 0.25 |
85
+ | qwen3 0.6B Q4_0 | 358.78 MiB | 596.05 M | CPU | 4 | 128 | 1 | 0 | tg128 | 11.73 ± 0.02 |
86
+
87
+
88
+ * Spacemit(R) A100
89
+ ~~~
90
+ model name : Spacemit(R) A100
91
+ isa : rv64imafdcvh_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintntl_zihintpause_zihpm_zimop_zaamo_zalrsc_zawrs_zfa_zfh_zfhmin_zca_zcb_zcd_zcmop_zba_zbb_zbc_zbs_zkt_zvbb_zvbc_zve32f_zve32x_zve64d_zve64f_zve64x_zvfh_zvfhmin_zvkb_zvkg_zvkned_zvknha_zvknhb_zvksed_zvksh_zvkt_smaia_smstateen_ssaia_sscofpmf_sstc_svinval_svnapot_svpbmt_sdtrig
92
+ mmu : sv39
93
+ mvendorid : 0x710
94
+ marchid : 0x8000000041000002
95
+ mimpid : 0x10000000d5686200
96
+ hart isa : rv64imafdcv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintntl_zihintpause_zihpm_zimop_zaamo_zalrsc_zawrs_zfa_zfh_zfhmin_zca_zcb_zcd_zcmop_zba_zbb_zbc_zbs_zkt_zvbb_zvbc_zve32f_zve32x_zve64d_zve64f_zve64x_zvfh_zvfhmin_zvkb_zvkg_zvkned_zvknha_zvknhb_zvksed_zvksh_zvkt_smaia_smstateen_ssaia_sscofpmf_sstc_svinval_svnapot_svpbmt_sdtrig
97
+ ~~~
98
+
99
+ | model | size | params | backend | threads | n_ubatch | fa | mmap | test | t/s |
100
+ | ------------------------------ | ---------: | ---------: | ---------- | ------: | -------: | -: | ---: | --------------: | -------------------: |
101
+ | qwen3 0.6B Q4_0 | 358.78 MiB | 596.05 M | CPU | 8 | 128 | 1 | 0 | pp128 | 565.83 ± 0.31 |
102
+ | qwen3 0.6B Q4_0 | 358.78 MiB | 596.05 M | CPU | 8 | 128 | 1 | 0 | tg128 | 55.77 ± 0.02 |
103
+ | qwen3 4B Q4_0 | 2.21 GiB | 4.02 B | CPU | 8 | 128 | 1 | 0 | pp128 | 79.74 ± 0.04 |
104
+ | qwen3 4B Q4_0 | 2.21 GiB | 4.02 B | CPU | 8 | 128 | 1 | 0 | tg128 | 11.29 ± 0.00 |
105
+ | qwen3moe 30B.A3B Q4_0 | 16.18 GiB | 30.53 B | CPU | 8 | 128 | 1 | 0 | pp128 | 57.88 ± 0.31 |
106
+ | qwen3moe 30B.A3B Q4_0 | 16.18 GiB | 30.53 B | CPU | 8 | 128 | 1 | 0 | tg128 | 12.79 ± 0.00 |
107
+ | qwen35 2B Q4_1 | 1.19 GiB | 1.88 B | CPU | 8 | 128 | 1 | 0 | pp128 | 115.23 ± 0.04 |
108
+ | qwen35 2B Q4_1 | 1.19 GiB | 1.88 B | CPU | 8 | 128 | 1 | 0 | tg128 | 16.49 ± 0.01 |
109
+ | gemma4 E4B Q4_K - Medium | 4.76 GiB | 7.52 B | CPU | 8 | 128 | 1 | 0 | pp128 | 21.13 ± 0.01 |
110
+ | gemma4 E4B Q4_K - Medium | 4.76 GiB | 7.52 B | CPU | 8 | 128 | 1 | 0 | tg128 | 5.66 ± 0.00 |
docs/build-s390x.md ADDED
@@ -0,0 +1,275 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ > [!IMPORTANT]
2
+ > This build documentation is specific only to IBM Z & LinuxONE mainframes (s390x). You can find the build documentation for other architectures: [build.md](build.md).
3
+
4
+ # Build llama.cpp locally (for s390x)
5
+
6
+ The main product of this project is the `llama` library. Its C-style interface can be found in [include/llama.h](../include/llama.h).
7
+
8
+ The project also includes many example programs and tools using the `llama` library. The examples range from simple, minimal code snippets to sophisticated sub-projects such as an OpenAI-compatible HTTP server.
9
+
10
+ **To get the code:**
11
+
12
+ ```bash
13
+ git clone https://github.com/ggml-org/llama.cpp
14
+ cd llama.cpp
15
+ ```
16
+
17
+ ## CPU Build with BLAS
18
+
19
+ Building llama.cpp with BLAS support is highly recommended as it has shown to provide performance improvements. Make sure to have OpenBLAS installed in your environment.
20
+
21
+ ```bash
22
+ cmake -S . -B build \
23
+ -DCMAKE_BUILD_TYPE=Release \
24
+ -DGGML_BLAS=ON \
25
+ -DGGML_BLAS_VENDOR=OpenBLAS
26
+
27
+ cmake --build build --config Release -j $(nproc)
28
+ ```
29
+
30
+ **Notes**:
31
+
32
+ - For faster repeated compilation, install [ccache](https://ccache.dev/)
33
+ - By default, VXE/VXE2 is enabled. To disable it (not recommended):
34
+
35
+ ```bash
36
+ cmake -S . -B build \
37
+ -DCMAKE_BUILD_TYPE=Release \
38
+ -DGGML_BLAS=ON \
39
+ -DGGML_BLAS_VENDOR=OpenBLAS \
40
+ -DGGML_VXE=OFF
41
+
42
+ cmake --build build --config Release -j $(nproc)
43
+ ```
44
+
45
+ - For debug builds:
46
+
47
+ ```bash
48
+ cmake -S . -B build \
49
+ -DCMAKE_BUILD_TYPE=Debug \
50
+ -DGGML_BLAS=ON \
51
+ -DGGML_BLAS_VENDOR=OpenBLAS
52
+ cmake --build build --config Debug -j $(nproc)
53
+ ```
54
+
55
+ - For static builds, add `-DBUILD_SHARED_LIBS=OFF`:
56
+
57
+ ```bash
58
+ cmake -S . -B build \
59
+ -DCMAKE_BUILD_TYPE=Release \
60
+ -DGGML_BLAS=ON \
61
+ -DGGML_BLAS_VENDOR=OpenBLAS \
62
+ -DBUILD_SHARED_LIBS=OFF
63
+
64
+ cmake --build build --config Release -j $(nproc)
65
+ ```
66
+
67
+ ## IBM zDNN Accelerator
68
+
69
+ This provides acceleration using the IBM zAIU co-processor located in the Telum I and Telum II processors. Make sure to have the [IBM zDNN library](https://github.com/IBM/zDNN) installed.
70
+
71
+ #### Compile from source from IBM
72
+
73
+ You may find the official build instructions here: [Building and Installing zDNN](https://github.com/IBM/zDNN?tab=readme-ov-file#building-and-installing-zdnn)
74
+
75
+ ### Compilation
76
+
77
+ ```bash
78
+ cmake -S . -B build \
79
+ -DCMAKE_BUILD_TYPE=Release \
80
+ -DGGML_ZDNN=ON
81
+ cmake --build build --config Release -j$(nproc)
82
+ ```
83
+
84
+ ## Getting GGUF Models
85
+
86
+ All models need to be converted to Big-Endian. You can achieve this in three cases:
87
+
88
+ 1. **Use pre-converted models verified for use on IBM Z & LinuxONE (easiest)**
89
+
90
+ ![File Type - gguf](https://img.shields.io/badge/File_Type-gguf-fff)
91
+
92
+ You can find popular models pre-converted and verified at [s390x Verified Models](https://huggingface.co/collections/taronaeo/s390x-verified-models-672765393af438d0ccb72a08) or [s390x Runnable Models](https://huggingface.co/collections/taronaeo/s390x-runnable-models-686e951824198df12416017e).
93
+
94
+ These models have already been converted from `safetensors` to `GGUF` Big-Endian and their respective tokenizers verified to run correctly on IBM z15 and later system.
95
+
96
+ 2. **Convert safetensors model to GGUF Big-Endian directly (recommended)**
97
+
98
+ ![File Type - safetensors](https://img.shields.io/badge/File_Type-safetensors-da1e28)
99
+
100
+ The model you are trying to convert must be in `safetensors` file format (for example [IBM Granite 3.3 2B](https://huggingface.co/ibm-granite/granite-3.3-2b-instruct)). Make sure you have downloaded the model repository for this case.
101
+
102
+ Ensure that you have installed the required packages in advance
103
+
104
+ ```bash
105
+ pip3 install -r requirements.txt
106
+ ```
107
+
108
+ Convert the `safetensors` model to `GGUF`
109
+
110
+ ```bash
111
+ python3 convert_hf_to_gguf.py \
112
+ --outfile model-name-be.f16.gguf \
113
+ --outtype f16 \
114
+ --bigendian \
115
+ model-directory/
116
+ ```
117
+
118
+ For example,
119
+
120
+ ```bash
121
+ python3 convert_hf_to_gguf.py \
122
+ --outfile granite-3.3-2b-instruct-be.f16.gguf \
123
+ --outtype f16 \
124
+ --bigendian \
125
+ granite-3.3-2b-instruct/
126
+ ```
127
+
128
+ 3. **Convert existing GGUF Little-Endian model to Big-Endian**
129
+
130
+ ![File Type - gguf](https://img.shields.io/badge/File_Type-gguf-fff)
131
+
132
+ The model you are trying to convert must be in `gguf` file format (for example [IBM Granite 3.3 2B GGUF](https://huggingface.co/ibm-granite/granite-3.3-2b-instruct-GGUF)). Make sure you have downloaded the model file for this case.
133
+
134
+ ```bash
135
+ python3 gguf-py/gguf/scripts/gguf_convert_endian.py model-name.f16.gguf BIG
136
+ ```
137
+
138
+ For example,
139
+
140
+ ```bash
141
+ python3 gguf-py/gguf/scripts/gguf_convert_endian.py granite-3.3-2b-instruct-le.f16.gguf BIG
142
+ mv granite-3.3-2b-instruct-le.f16.gguf granite-3.3-2b-instruct-be.f16.gguf
143
+ ```
144
+
145
+ **Notes:**
146
+
147
+ - The GGUF endian conversion script may not support all data types at the moment and may fail for some models/quantizations. When that happens, please try manually converting the safetensors model to GGUF Big-Endian via Step 2.
148
+
149
+ ## IBM Accelerators
150
+
151
+ ### 1. SIMD Acceleration
152
+
153
+ Only available in IBM z15/LinuxONE 3 or later system with the `-DGGML_VXE=ON` (turned on by default) compile flag. No hardware acceleration is possible with llama.cpp with older systems, such as IBM z14/arch12. In such systems, the APIs can still run but will use a scalar implementation.
154
+
155
+ ### 2. zDNN Accelerator (WIP)
156
+
157
+ Only available in IBM z17/LinuxONE 5 or later system with the `-DGGML_ZDNN=ON` compile flag. No hardware acceleration is possible with llama.cpp with older systems, such as IBM z15/arch13. In such systems, the APIs will default back to CPU routines.
158
+
159
+ ### 3. Spyre Accelerator
160
+
161
+ _Only available with IBM z17 / LinuxONE 5 or later system. No support currently available._
162
+
163
+ ## Performance Tuning
164
+
165
+ ### 1. Virtualization Setup
166
+
167
+ It is strongly recommended to use only LPAR (Type-1) virtualization to get the most performance.
168
+
169
+ Note: Type-2 virtualization is not supported at the moment, while you can get it running, the performance will not be the best.
170
+
171
+ ### 2. IFL (Core) Count
172
+
173
+ It is recommended to allocate a minimum of 8 shared IFLs assigned to the LPAR. Increasing the IFL count past 8 shared IFLs will only improve Prompt Processing performance but not Token Generation.
174
+
175
+ Note: IFL count does not equate to vCPU count.
176
+
177
+ ### 3. SMT vs NOSMT (Simultaneous Multithreading)
178
+
179
+ It is strongly recommended to disable SMT via the kernel boot parameters as it negatively affects performance. Please refer to your Linux distribution's guide on disabling SMT via kernel boot parameters.
180
+
181
+ ### 4. BLAS vs NOBLAS
182
+
183
+ IBM VXE/VXE2 SIMD acceleration depends on the BLAS implementation. It is strongly recommended to use BLAS.
184
+
185
+ ## Frequently Asked Questions (FAQ)
186
+
187
+ 1. I'm getting the following error message while trying to load a model: `gguf_init_from_file_impl: failed to load model: this GGUF file version 50331648 is extremely large, is there a mismatch between the host and model endianness?`
188
+
189
+ Answer: Please ensure that the model you have downloaded/converted is GGUFv3 Big-Endian. These models are usually denoted with the `-be` suffix, i.e., `granite-3.3-2b-instruct-be.F16.gguf`.
190
+
191
+ You may refer to the [Getting GGUF Models](#getting-gguf-models) section to manually convert a `safetensors` model to `GGUF` Big Endian.
192
+
193
+ 2. I'm getting extremely poor performance when running inference on a model
194
+
195
+ Answer: Please refer to the [Appendix B: SIMD Support Matrix](#appendix-b-simd-support-matrix) to check if your model quantization is supported by SIMD acceleration.
196
+
197
+ 3. I'm building on IBM z17 and getting the following error messages: `invalid switch -march=z17`
198
+
199
+ Answer: Please ensure that your GCC compiler is of minimum GCC 15.1.0 version, and have `binutils` updated to the latest version. If this does not fix the problem, kindly open an issue.
200
+
201
+ 4. Failing to install the `sentencepiece` package using GCC 15+
202
+
203
+ Answer: The `sentencepiece` team are aware of this as seen in [this issue](https://github.com/google/sentencepiece/issues/1108).
204
+
205
+ As a temporary workaround, please run the installation command with the following environment variables.
206
+
207
+ ```bash
208
+ export CXXFLAGS="-include cstdint"
209
+ ```
210
+
211
+ For example,
212
+
213
+ ```bash
214
+ CXXFLAGS="-include cstdint" pip3 install -r requirements.txt
215
+ ```
216
+
217
+ ## Getting Help on IBM Z & LinuxONE
218
+
219
+ 1. **Bugs, Feature Requests**
220
+
221
+ Please file an issue in llama.cpp and ensure that the title contains "s390x".
222
+
223
+ 2. **Other Questions**
224
+
225
+ Please reach out directly to [aionz@us.ibm.com](mailto:aionz@us.ibm.com).
226
+
227
+ ## Appendix A: Hardware Support Matrix
228
+
229
+ | | Support | Minimum Compiler Version |
230
+ | -------- | ------- | ------------------------ |
231
+ | IBM z15 | ✅ | |
232
+ | IBM z16 | ✅ | |
233
+ | IBM z17 | ✅ | GCC 15.1.0 |
234
+ | IBM zDNN | ✅ | |
235
+
236
+ - ✅ - supported and verified to run as intended
237
+ - 🚫 - unsupported, we are unlikely able to provide support
238
+
239
+ ## Appendix B: SIMD Support Matrix
240
+
241
+ | | VX/VXE/VXE2 | zDNN | Spyre |
242
+ |------------|-------------|------|-------|
243
+ | FP32 | ✅ | ✅ | ❓ |
244
+ | FP16 | ✅ | ✅ | ❓ |
245
+ | BF16 | ✅ | ✅ | ❓ |
246
+ | Q4_0 | ✅ | ❓ | ❓ |
247
+ | Q4_1 | ✅ | ❓ | ❓ |
248
+ | MXFP4 | ✅ | ❓ | ❓ |
249
+ | Q5_0 | ✅ | ❓ | ❓ |
250
+ | Q5_1 | ✅ | ❓ | ❓ |
251
+ | Q8_0 | ✅ | ❓ | ❓ |
252
+ | Q2_K | 🚫 | ❓ | ❓ |
253
+ | Q3_K | ✅ | ❓ | ❓ |
254
+ | Q4_K | ✅ | ❓ | ❓ |
255
+ | Q5_K | ✅ | ❓ | ❓ |
256
+ | Q6_K | ✅ | ❓ | ❓ |
257
+ | TQ1_0 | 🚫 | ❓ | ❓ |
258
+ | TQ2_0 | 🚫 | ❓ | ❓ |
259
+ | IQ2_XXS | 🚫 | ❓ | ❓ |
260
+ | IQ2_XS | 🚫 | ❓ | ❓ |
261
+ | IQ2_S | 🚫 | ❓ | ❓ |
262
+ | IQ3_XXS | 🚫 | ❓ | ❓ |
263
+ | IQ3_S | 🚫 | ❓ | ❓ |
264
+ | IQ1_S | 🚫 | ❓ | ❓ |
265
+ | IQ1_M | 🚫 | ❓ | ❓ |
266
+ | IQ4_NL | ✅ | ❓ | ❓ |
267
+ | IQ4_XS | ✅ | ❓ | ❓ |
268
+ | FP32->FP16 | 🚫 | ❓ | ❓ |
269
+ | FP16->FP32 | 🚫 | ❓ | ❓ |
270
+
271
+ - ✅ - acceleration available
272
+ - 🚫 - acceleration unavailable, will still run using scalar implementation
273
+ - ❓ - acceleration unknown, please contribute if you can test it yourself
274
+
275
+ Last Updated by **Aaron Teo (aaron.teo1@ibm.com)** on Feb 15, 2026.
docs/build.md ADDED
@@ -0,0 +1,844 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Build llama.cpp locally
2
+
3
+ The main product of this project is the `llama` library. Its C-style interface can be found in [include/llama.h](../include/llama.h).
4
+
5
+ The project also includes many example programs and tools using the `llama` library. The examples range from simple, minimal code snippets to sophisticated sub-projects such as an OpenAI-compatible HTTP server.
6
+
7
+ **To get the Code:**
8
+
9
+ ```bash
10
+ git clone https://github.com/ggml-org/llama.cpp
11
+ cd llama.cpp
12
+ ```
13
+
14
+ The following sections describe how to build with different backends and options.
15
+
16
+ * [CPU Build](#cpu-build)
17
+ * [BLAS Build](#blas-build)
18
+ * [Metal Build](#metal-build)
19
+ * [SYCL](#sycl)
20
+ * [CUDA](#cuda)
21
+ * [MUSA](#musa)
22
+ * [HIP](#hip)
23
+ * [Vulkan](#vulkan)
24
+ * [CANN](#cann)
25
+ * [ZenDNN](#zendnn)
26
+ * [Arm® KleidiAI™](#arm-kleidiai)
27
+ * [OpenCL](#opencl)
28
+ * [Android](#android-1)
29
+ * [OpenVINO](#openvino)
30
+ * [Hexagon](#hexagon)
31
+ * [Notes about GPU-accelerated backends](#notes-about-gpu-accelerated-backends)
32
+
33
+ ## CPU Build
34
+
35
+ Build llama.cpp using `CMake`:
36
+
37
+ ```bash
38
+ cmake -B build
39
+ cmake --build build --config Release
40
+ ```
41
+
42
+ **Notes**:
43
+
44
+ - For faster compilation, add the `-j` argument to run multiple jobs in parallel, or use a generator that does this automatically such as Ninja. For example, `cmake --build build --config Release -j 8` will run 8 jobs in parallel.
45
+ - For faster repeated compilation, install [ccache](https://ccache.dev/)
46
+ - For debug builds, there are two cases:
47
+
48
+ 1. Single-config generators (e.g. default = `Unix Makefiles`; note that they just ignore the `--config` flag):
49
+
50
+ ```bash
51
+ cmake -B build -DCMAKE_BUILD_TYPE=Debug
52
+ cmake --build build
53
+ ```
54
+
55
+ 2. Multi-config generators (`-G` param set to Visual Studio, XCode...):
56
+
57
+ ```bash
58
+ cmake -B build -G "Xcode"
59
+ cmake --build build --config Debug
60
+ ```
61
+
62
+ For more details and a list of supported generators, see the [CMake documentation](https://cmake.org/cmake/help/latest/manual/cmake-generators.7.html).
63
+ - For static builds, add `-DBUILD_SHARED_LIBS=OFF`:
64
+ ```
65
+ cmake -B build -DBUILD_SHARED_LIBS=OFF
66
+ cmake --build build --config Release
67
+ ```
68
+
69
+ - Building for Windows (x86, x64 and arm64) with MSVC or clang as compilers:
70
+ - Install Visual Studio 2022, e.g. via the [Community Edition](https://visualstudio.microsoft.com/vs/community/). In the installer, select at least the following options (this also automatically installs the required additional tools like CMake,...):
71
+ - Tab Workload: Desktop-development with C++
72
+ - Tab Components (select quickly via search): C++-_CMake_ Tools for Windows, _Git_ for Windows, C++-_Clang_ Compiler for Windows, MS-Build Support for LLVM-Toolset (clang)
73
+ - Please remember to always use a Developer Command Prompt / PowerShell for VS2022 for git, build, test
74
+ - For Windows on ARM (arm64, WoA), build with:
75
+ ```bash
76
+ cmake --preset arm64-windows-llvm-release -D GGML_OPENMP_FETCH=ON
77
+ cmake --build build-arm64-windows-llvm-release
78
+ ```
79
+ - Use `ARM64 Native Tools Command Prompt for VS 2022` if you are building on an ARM64 machine.
80
+ - `GGML_OPENMP_FETCH` downloads the official LLVM OpenMP runtime and requires Clang, 7-Zip and network access during configuration. CMake selects the runtime from the target architecture, so this also works when cross-compiling for WoA from x64. The extracted header, import library, DLL and OpenMP license are placed under `build/_deps`. The build copies `libomp.dll` and `LICENSE-LLVM-OpenMP` to the runtime output directory and installs them together. Omit the option to use CMake's normal OpenMP detection, or pass `-D GGML_OPENMP=OFF` to disable OpenMP.
81
+ - For building with ninja generator and clang compiler as default:
82
+ - Set path:
83
+ ```
84
+ set LIB=C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\um\x64;C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Tools\MSVC\14.41.34120\lib\x64\uwp;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\ucrt\x64
85
+ ```
86
+ - Run:
87
+ ```bash
88
+ cmake --preset x64-windows-llvm-release
89
+ cmake --build build-x64-windows-llvm-release
90
+ ```
91
+ - If you want HTTPS/TLS features, you may install OpenSSL development libraries. If not installed, the project will build and run without SSL support.
92
+ - **Debian / Ubuntu:** `sudo apt-get install libssl-dev`
93
+ - **Fedora / RHEL / Rocky / Alma:** `sudo dnf install openssl-devel`
94
+ - **Arch / Manjaro:** `sudo pacman -S openssl`
95
+
96
+ ## BLAS Build
97
+
98
+ Building the program with BLAS support may lead to some performance improvements in prompt processing using batch sizes higher than 32 (the default is 512). Using BLAS doesn't affect the generation performance. There are currently several different BLAS implementations available for build and use:
99
+
100
+ ### Accelerate Framework
101
+
102
+ This is only available on Mac PCs and it's enabled by default. You can just build using the normal instructions.
103
+
104
+ ### OpenBLAS
105
+
106
+ This provides BLAS acceleration using only the CPU. Make sure to have OpenBLAS installed on your machine.
107
+
108
+ - Using `CMake` on Linux:
109
+
110
+ ```bash
111
+ cmake -B build -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS
112
+ cmake --build build --config Release
113
+ ```
114
+
115
+ ### BLIS
116
+
117
+ Check [BLIS.md](./backend/BLIS.md) for more information.
118
+
119
+ ### Intel oneMKL
120
+
121
+ Building through oneAPI compilers will make avx_vnni instruction set available for intel processors that do not support avx512 and avx512_vnni. Please note that this build config **does not support Intel GPU**. For Intel GPU support, please refer to [llama.cpp for SYCL](./backend/SYCL.md).
122
+
123
+ - Using manual oneAPI installation:
124
+ By default, `GGML_BLAS_VENDOR` is set to `Generic`, so if you already sourced intel environment script and assign `-DGGML_BLAS=ON` in cmake, the mkl version of Blas will automatically been selected. Otherwise please install oneAPI and follow the below steps:
125
+ ```bash
126
+ source /opt/intel/oneapi/setvars.sh # You can skip this step if in oneapi-basekit docker image, only required for manual installation
127
+ cmake -B build -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=Intel10_64lp -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_NATIVE=ON
128
+ cmake --build build --config Release
129
+ ```
130
+
131
+ - Using oneAPI docker image:
132
+ If you do not want to source the environment vars and install oneAPI manually, you can also build the code using intel docker container: [oneAPI-basekit](https://hub.docker.com/r/intel/oneapi-basekit). Then, you can use the commands given above.
133
+
134
+ Check [Optimizing and Running LLaMA2 on Intel® CPU](https://builders.intel.com/solutionslibrary/optimizing-and-running-llama2-on-intel-cpu) for more information.
135
+
136
+ ### Other BLAS libraries
137
+
138
+ Any other BLAS library can be used by setting the `GGML_BLAS_VENDOR` option. See the [CMake documentation](https://cmake.org/cmake/help/latest/module/FindBLAS.html#blas-lapack-vendors) for a list of supported vendors.
139
+
140
+ ## Metal Build
141
+
142
+ On MacOS, Metal is enabled by default. Using Metal makes the computation run on the GPU.
143
+ To disable the Metal build at compile time use the `-DGGML_METAL=OFF` cmake option.
144
+
145
+ When built with Metal support, you can explicitly disable GPU inference with the `--n-gpu-layers 0` command-line argument.
146
+
147
+ ## SYCL
148
+
149
+ SYCL is a higher-level programming model to improve programming productivity on various hardware accelerators.
150
+
151
+ llama.cpp based on SYCL is used to **support Intel GPU** (Data Center Max series, Flex series, Arc series, Built-in GPU and iGPU).
152
+
153
+ For detailed info, please refer to [llama.cpp for SYCL](./backend/SYCL.md).
154
+
155
+ ## CUDA
156
+
157
+ This provides GPU acceleration using an NVIDIA GPU. Make sure to have the [CUDA toolkit](https://developer.nvidia.com/cuda-toolkit) installed.
158
+
159
+ #### Download directly from NVIDIA
160
+ You may find the official downloads here: [NVIDIA developer site](https://developer.nvidia.com/cuda-downloads).
161
+
162
+
163
+ #### Compile and run inside a Fedora Toolbox Container
164
+ We also have a [guide](./backend/CUDA-FEDORA.md) for setting up CUDA toolkit in a Fedora [toolbox container](https://containertoolbx.org/).
165
+
166
+ **Recommended for:**
167
+ - ***Necessary*** for users of [Atomic Desktops for Fedora](https://fedoraproject.org/atomic-desktops/); such as: [Silverblue](https://fedoraproject.org/atomic-desktops/silverblue/) and [Kinoite](https://fedoraproject.org/atomic-desktops/kinoite/).
168
+ - (there are no supported CUDA packages for these systems)
169
+ - ***Necessary*** for users that have a host that is not a: [Supported Nvidia CUDA Release Platform](https://developer.nvidia.com/cuda-downloads).
170
+ - (for example, you may have [Fedora 42 Beta](https://fedoramagazine.org/announcing-fedora-linux-42-beta/) as your host operating system)
171
+ - ***Convenient*** For those running [Fedora Workstation](https://fedoraproject.org/workstation/) or [Fedora KDE Plasma Desktop](https://fedoraproject.org/spins/kde), and want to keep their host system clean.
172
+ - *Optionally* toolbox packages are available: [Arch Linux](https://archlinux.org/), [Red Hat Enterprise Linux >= 8.5](https://www.redhat.com/en/technologies/linux-platforms/enterprise-linux), or [Ubuntu](https://ubuntu.com/download)
173
+
174
+
175
+ ### Compilation
176
+
177
+ Make sure to read the notes about the CPU build for general instructions for e.g. speeding up the compilation.
178
+
179
+ ```bash
180
+ cmake -B build -DGGML_CUDA=ON
181
+ cmake --build build --config Release
182
+ ```
183
+
184
+ ### Non-Native Builds
185
+
186
+ By default llama.cpp will be built for the hardware that is connected to the system at that time.
187
+ For a build covering all CUDA GPUs, disable `GGML_NATIVE`:
188
+
189
+ ```bash
190
+ cmake -B build -DGGML_CUDA=ON -DGGML_NATIVE=OFF
191
+ ```
192
+
193
+ The resulting binary should run on all CUDA GPUs with optimal performance, though some just-in-time compilation may be required.
194
+
195
+ ### Override Compute Capability Specifications
196
+
197
+ If `nvcc` cannot detect your gpu, you may get compile warnings such as:
198
+ ```text
199
+ nvcc warning : Cannot find valid GPU for '-arch=native', default arch is used
200
+ ```
201
+
202
+ One option is to do a non-native build as described above.
203
+ However, this will result in a large binary that takes a long time to compile.
204
+ Alternatively it is also possible to explicitly specify CUDA architectures.
205
+ This may also make sense for a non-native build, for that one should look at the logic in `ggml/src/ggml-cuda/CMakeLists.txt` as a starting point.
206
+
207
+ To override the default CUDA architectures:
208
+
209
+ #### 1. Take note of the `Compute Capability` of your NVIDIA devices: ["CUDA: Your GPU Compute > Capability"](https://developer.nvidia.com/cuda-gpus).
210
+
211
+ ```text
212
+ GeForce RTX 4090 8.9
213
+ GeForce RTX 3080 Ti 8.6
214
+ GeForce RTX 3070 8.6
215
+ ```
216
+
217
+ #### 2. Manually list each varying `Compute Capability` in the `CMAKE_CUDA_ARCHITECTURES` list.
218
+
219
+ ```bash
220
+ cmake -B build -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES="86;89"
221
+ ```
222
+
223
+ ### Overriding the CUDA Version
224
+
225
+ If you have multiple CUDA installations on your system and want to compile llama.cpp for a specific one, e.g. for CUDA 11.7 installed under `/opt/cuda-11.7`:
226
+
227
+ ```bash
228
+ cmake -B build -DGGML_CUDA=ON -DCMAKE_CUDA_COMPILER=/opt/cuda-11.7/bin/nvcc -DCMAKE_INSTALL_RPATH="/opt/cuda-11.7/lib64;\$ORIGIN" -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON
229
+ ```
230
+
231
+ #### Fixing Compatibility Issues with Old CUDA and New glibc
232
+
233
+ If you try to use an old CUDA version (e.g. v11.7) with a new glibc version you can get errors like this:
234
+
235
+ ```
236
+ /usr/include/bits/mathcalls.h(83): error: exception specification is
237
+ incompatible with that of previous function "cospi"
238
+
239
+
240
+ /opt/cuda-11.7/bin/../targets/x86_64-linux/include/crt/math_functions.h(5545):
241
+ here
242
+ ```
243
+
244
+ It seems the least bad solution is to patch the CUDA installation to declare the correct signatures.
245
+ Replace the following lines in `/path/to/your/cuda/installation/targets/x86_64-linux/include/crt/math_functions.h`:
246
+
247
+ ```C++
248
+ // original lines
249
+ extern __DEVICE_FUNCTIONS_DECL__ __device_builtin__ double cospi(double x);
250
+ extern __DEVICE_FUNCTIONS_DECL__ __device_builtin__ float cospif(float x);
251
+ extern __DEVICE_FUNCTIONS_DECL__ __device_builtin__ double sinpi(double x);
252
+ extern __DEVICE_FUNCTIONS_DECL__ __device_builtin__ float sinpif(float x);
253
+ extern __DEVICE_FUNCTIONS_DECL__ __device_builtin__ double rsqrt(double x);
254
+ extern __DEVICE_FUNCTIONS_DECL__ __device_builtin__ float rsqrtf(float x);
255
+
256
+ // edited lines
257
+ extern __DEVICE_FUNCTIONS_DECL__ __device_builtin__ double cospi(double x) noexcept (true);
258
+ extern __DEVICE_FUNCTIONS_DECL__ __device_builtin__ float cospif(float x) noexcept (true);
259
+ extern __DEVICE_FUNCTIONS_DECL__ __device_builtin__ double sinpi(double x) noexcept (true);
260
+ extern __DEVICE_FUNCTIONS_DECL__ __device_builtin__ float sinpif(float x) noexcept (true);
261
+ extern __DEVICE_FUNCTIONS_DECL__ __device_builtin__ double rsqrt(double x) noexcept (true);
262
+ extern __DEVICE_FUNCTIONS_DECL__ __device_builtin__ float rsqrtf(float x) noexcept (true);
263
+ ```
264
+
265
+ ### Runtime CUDA environmental variables
266
+
267
+ You may set the [cuda environmental variables](https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#env-vars) at runtime.
268
+
269
+ ```bash
270
+ # Use `CUDA_VISIBLE_DEVICES` to hide the first compute device.
271
+ CUDA_VISIBLE_DEVICES="-0" ./build/bin/llama-server --model /srv/models/llama.gguf
272
+ ```
273
+
274
+ #### CUDA_SCALE_LAUNCH_QUEUES
275
+
276
+ The environment variable [`CUDA_SCALE_LAUNCH_QUEUES`](https://docs.nvidia.com/cuda/cuda-programming-guide/05-appendices/environment-variables.html#cuda-scale-launch-queues) controls the size of CUDA's command buffer, which determines how many GPU operations can be queued before the CPU must wait for the GPU to catch up. A larger buffer reduces CPU-side stalls and allows more work to be queued on a GPU.
277
+
278
+ Consider setting `CUDA_SCALE_LAUNCH_QUEUES=4x`, which increases the CUDA command buffer to 4 times its default size. This optimization is particularly beneficial for **Multi-GPU setups with pipeline parallelism**, where it significantly improves prompt processing throughput by allowing more operations to be enqueued across GPUs.
279
+
280
+ #### GGML_CUDA_CUBLAS_COMPUTE_TYPE
281
+
282
+ Override default, speed-optimized compute types for cuBLAS matrix multiplications.
283
+ Legal values: `auto`, `f16`, `fp16`, `bf16`, `f32`, `fp32`.
284
+
285
+ ### Unified Memory
286
+
287
+ The environment variable `GGML_CUDA_ENABLE_UNIFIED_MEMORY=1` can be used to enable unified memory in Linux. This allows swapping to system RAM instead of crashing when the GPU VRAM is exhausted. In Windows this setting is available in the NVIDIA control panel as `System Memory Fallback`.
288
+
289
+ ### Peer Access
290
+
291
+ The environment variable `GGML_CUDA_P2P` can be set to enable peer-to-peer access between multiple GPUs, allowing them to transfer data directly rather than to go through system memory.
292
+ Requires driver support (usually restricted to workstation/datacenter GPUs).
293
+ May cause crashes or corrupted outputs for some motherboards and BIOS settings (e.g. IOMMU).
294
+
295
+ ### Performance Tuning
296
+
297
+ The following compilation options are also available to tweak performance:
298
+
299
+ | Option | Legal values | Default | Description |
300
+ |-------------------------------|------------------------|---------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
301
+ | GGML_CUDA_FORCE_MMQ | Boolean | false | Force the use of custom matrix multiplication kernels for quantized models instead of FP16 cuBLAS even if there is no int8 tensor core implementation available (affects V100, CDNA and RDNA3+). MMQ kernels are enabled by default on GPUs with int8 tensor core support. With MMQ force enabled, speed for large batch sizes will be worse but VRAM consumption will be lower. |
302
+ | GGML_CUDA_FORCE_CUBLAS | Boolean | false | Force the use of FP16 cuBLAS instead of custom matrix multiplication kernels for quantized models. There may be issues with numerical overflows (except for V100, CDNA and RDNA4 which use FP32 compute type by default) and memory use will be higher. Prompt processing may become faster on recent datacenter GPUs (the custom kernels were tuned primarily for RTX 3000/4000). |
303
+ | GGML_CUDA_FA_ALL_QUANTS | Boolean | false | Compile support for all KV cache quantization type (combinations) for the FlashAttention CUDA kernels. More fine-grained control over KV cache size but compilation takes much longer. |
304
+
305
+ ## MUSA
306
+
307
+ This provides GPU acceleration using a Moore Threads GPU. Make sure to have the [MUSA SDK](https://developer.mthreads.com/musa/musa-sdk) installed.
308
+
309
+ #### Download directly from Moore Threads
310
+
311
+ You may find the official downloads here: [Moore Threads developer site](https://developer.mthreads.com/sdk/download/musa).
312
+
313
+ ### Compilation
314
+
315
+ ```bash
316
+ cmake -B build -DGGML_MUSA=ON
317
+ cmake --build build --config Release
318
+ ```
319
+
320
+ #### Override Compute Capability Specifications
321
+
322
+ By default, all supported compute capabilities are enabled. To customize this behavior, you can specify the `MUSA_ARCHITECTURES` option in the CMake command:
323
+
324
+ ```bash
325
+ cmake -B build -DGGML_MUSA=ON -DMUSA_ARCHITECTURES="21"
326
+ cmake --build build --config Release
327
+ ```
328
+
329
+ This configuration enables only compute capability `2.1` (MTT S80) during compilation, which can help reduce compilation time.
330
+
331
+ #### Compilation options
332
+
333
+ Most of the compilation options available for CUDA should also be available for MUSA, though they haven't been thoroughly tested yet.
334
+
335
+ - For static builds, add `-DBUILD_SHARED_LIBS=OFF` and `-DCMAKE_POSITION_INDEPENDENT_CODE=ON`:
336
+ ```
337
+ cmake -B build -DGGML_MUSA=ON \
338
+ -DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON
339
+ cmake --build build --config Release
340
+ ```
341
+
342
+ ### Runtime MUSA environmental variables
343
+
344
+ You may set the [musa environmental variables](https://docs.mthreads.com/musa-sdk/musa-sdk-doc-online/programming_guide/Z%E9%99%84%E5%BD%95/) at runtime.
345
+
346
+ ```bash
347
+ # Use `MUSA_VISIBLE_DEVICES` to hide the first compute device.
348
+ MUSA_VISIBLE_DEVICES="-0" ./build/bin/llama-server --model /srv/models/llama.gguf
349
+ ```
350
+
351
+ ### Unified Memory
352
+
353
+ The environment variable `GGML_CUDA_ENABLE_UNIFIED_MEMORY=1` can be used to enable unified memory in Linux. This allows swapping to system RAM instead of crashing when the GPU VRAM is exhausted.
354
+
355
+ ## HIP
356
+
357
+ This provides GPU acceleration on HIP-supported AMD GPUs.
358
+ Make sure to have ROCm installed.
359
+ You can download it from your Linux distro's package manager or from here: [ROCm Quick Start (Linux)](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/tutorial/quick-start.html#rocm-install-quick).
360
+
361
+ - Using `CMake` for Linux (assuming a gfx1030-compatible AMD GPU):
362
+ ```bash
363
+ HIPCXX="$(hipconfig -l)/clang" HIP_PATH="$(hipconfig -R)" \
364
+ cmake -S . -B build -DGGML_HIP=ON -DGPU_TARGETS=gfx1030 -DCMAKE_BUILD_TYPE=Release \
365
+ && cmake --build build --config Release -- -j 16
366
+ ```
367
+
368
+ Note: `GPU_TARGETS` is optional, omitting it will build the code for all GPUs in the current system.
369
+
370
+ Note that if you get the following error:
371
+ ```
372
+ clang: error: cannot find ROCm device library; provide its path via '--rocm-path' or '--rocm-device-lib-path', or pass '-nogpulib' to build without ROCm device library
373
+ ```
374
+ Try searching for a directory under `HIP_PATH` that contains the file
375
+ `oclc_abi_version_400.bc`. Then, add the following to the start of the
376
+ command: `HIP_DEVICE_LIB_PATH=<directory-you-just-found>`, so something
377
+ like:
378
+ ```bash
379
+ HIPCXX="$(hipconfig -l)/clang" HIP_PATH="$(hipconfig -p)" \
380
+ HIP_DEVICE_LIB_PATH=<directory-you-just-found> \
381
+ cmake -S . -B build -DGGML_HIP=ON -DGPU_TARGETS=gfx1030 -DCMAKE_BUILD_TYPE=Release \
382
+ && cmake --build build -- -j 16
383
+ ```
384
+
385
+ - Using `CMake` for Windows (using x64 Native Tools Command Prompt for VS, and assuming a gfx1100-compatible AMD GPU):
386
+ ```bash
387
+ set PATH=%HIP_PATH%\bin;%PATH%
388
+ cmake -S . -B build -G Ninja -DGPU_TARGETS=gfx1100 -DGGML_HIP=ON -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=Release
389
+ cmake --build build
390
+ ```
391
+ If necessary, adapt `GPU_TARGETS` to the GPU arch you want to compile for. The above example uses `gfx1100` that corresponds to Radeon RX 7900XTX/XT/GRE. You can find a list of targets [here](https://llvm.org/docs/AMDGPUUsage.html#processors)
392
+ Find your gpu version string by matching the most significant version information from `rocminfo | grep gfx | head -1 | awk '{print $2}'` with the list of processors, e.g. `gfx1035` maps to `gfx1030`.
393
+
394
+
395
+ The environment variable [`HIP_VISIBLE_DEVICES`](https://rocm.docs.amd.com/en/latest/understand/gpu_isolation.html#hip-visible-devices) can be used to specify which GPU(s) will be used.
396
+ If your GPU is not officially supported you can use the environment variable [`HSA_OVERRIDE_GFX_VERSION`] set to a similar GPU, for example 10.3.0 on RDNA2 (e.g. gfx1030, gfx1031, or gfx1035) or 11.0.0 on RDNA3. Note that [`HSA_OVERRIDE_GFX_VERSION`] is [not supported on Windows](https://github.com/ROCm/ROCm/issues/2654)
397
+
398
+ ### Unified Memory
399
+
400
+ On Linux it is possible to use unified memory architecture (UMA) to share main memory between the CPU and integrated GPU by setting environment variable `GGML_CUDA_ENABLE_UNIFIED_MEMORY=1`. However, this hurts performance for non-integrated GPUs (but enables working with integrated GPUs).
401
+
402
+ ## Vulkan
403
+
404
+ ### For Windows Users:
405
+ **w64devkit**
406
+
407
+ Download and extract [`w64devkit`](https://github.com/skeeto/w64devkit/releases).
408
+
409
+ Download and install the [`Vulkan SDK`](https://vulkan.lunarg.com/sdk/home#windows) with the default settings.
410
+
411
+ Launch `w64devkit.exe` and run the following commands to copy Vulkan dependencies:
412
+ ```sh
413
+ SDK_VERSION=1.3.283.0
414
+ cp /VulkanSDK/$SDK_VERSION/Bin/glslc.exe $W64DEVKIT_HOME/bin/
415
+ cp /VulkanSDK/$SDK_VERSION/Lib/vulkan-1.lib $W64DEVKIT_HOME/x86_64-w64-mingw32/lib/
416
+ cp -r /VulkanSDK/$SDK_VERSION/Include/* $W64DEVKIT_HOME/x86_64-w64-mingw32/include/
417
+ cat > $W64DEVKIT_HOME/x86_64-w64-mingw32/lib/pkgconfig/vulkan.pc <<EOF
418
+ Name: Vulkan-Loader
419
+ Description: Vulkan Loader
420
+ Version: $SDK_VERSION
421
+ Libs: -lvulkan-1
422
+ EOF
423
+
424
+ ```
425
+
426
+ Switch into the `llama.cpp` directory and build using CMake.
427
+ ```sh
428
+ cmake -B build -DGGML_VULKAN=ON
429
+ cmake --build build --config Release
430
+ ```
431
+
432
+ **Git Bash MINGW64**
433
+
434
+ Download and install [`Git-SCM`](https://git-scm.com/downloads/win) with the default settings
435
+
436
+ Download and install [`Visual Studio Community Edition`](https://visualstudio.microsoft.com/) and make sure you select `C++`
437
+
438
+ Download and install [`CMake`](https://cmake.org/download/) with the default settings
439
+
440
+ Download and install the [`Vulkan SDK`](https://vulkan.lunarg.com/sdk/home#windows) with the default settings.
441
+
442
+ Go into your `llama.cpp` directory and right click, select `Open Git Bash Here` and then run the following commands
443
+
444
+ ```
445
+ cmake -B build -DGGML_VULKAN=ON
446
+ cmake --build build --config Release
447
+ ```
448
+
449
+ Now you can load the model in conversation mode using `Vulkan`
450
+
451
+ ```sh
452
+ build/bin/Release/llama-cli -m "[PATH TO MODEL]" -ngl 100 -c 16384 -t 10 -n -2 -cnv
453
+ ```
454
+
455
+ **MSYS2**
456
+
457
+ Install [MSYS2](https://www.msys2.org/) and then run the following commands in a UCRT terminal to install dependencies.
458
+ ```sh
459
+ pacman -S git \
460
+ mingw-w64-ucrt-x86_64-gcc \
461
+ mingw-w64-ucrt-x86_64-cmake \
462
+ mingw-w64-ucrt-x86_64-vulkan-devel \
463
+ mingw-w64-ucrt-x86_64-shaderc \
464
+ mingw-w64-ucrt-x86_64-spirv-headers
465
+ ```
466
+
467
+ Switch into the `llama.cpp` directory and build using CMake.
468
+ ```sh
469
+ cmake -B build -DGGML_VULKAN=ON
470
+ cmake --build build --config Release
471
+ ```
472
+
473
+ ### For Docker users:
474
+
475
+ You don't need to install the Vulkan SDK. It will be installed inside the container.
476
+
477
+ ```sh
478
+ # Build the image
479
+ docker build -t llama-cpp-vulkan --target light -f .devops/vulkan.Dockerfile .
480
+
481
+ # Then, use it:
482
+ docker run -it --rm -v "$(pwd):/app:Z" --device /dev/dri/renderD128:/dev/dri/renderD128 --device /dev/dri/card1:/dev/dri/card1 llama-cpp-vulkan -m "/app/models/YOUR_MODEL_FILE" -p "Building a website can be done in 10 simple steps:" -n 400 -e -ngl 33
483
+ ```
484
+
485
+ ### For Linux users:
486
+
487
+ #### Using the LunarG Vulkan SDK
488
+
489
+ First, follow the official LunarG instructions for the installation and setup of the Vulkan SDK in the [Getting Started with the Linux Tarball Vulkan SDK](https://vulkan.lunarg.com/doc/sdk/latest/linux/getting_started.html) guide.
490
+
491
+ > [!IMPORTANT]
492
+ > After completing the first step, ensure that you have used the `source` command on the `setup_env.sh` file inside of the Vulkan SDK in your current terminal session. Otherwise, the build won't work. Additionally, if you close out of your terminal, you must perform this step again if you intend to perform a build. However, there are ways to make this persistent. Refer to the Vulkan SDK guide linked in the first step for more information about any of this.
493
+
494
+ #### Using system packages
495
+
496
+ On Debian / Ubuntu, you can install the required dependencies using:
497
+ ```sh
498
+ sudo apt-get install libvulkan-dev glslc spirv-headers
499
+ ```
500
+
501
+ SPIRV-Headers (`spirv/unified1/spirv.hpp`) are required for the Vulkan backend and are **not** always pulled in by the Vulkan loader dev package alone. Other distros use names such as `spirv-headers` (Ubuntu / Debian / Arch), or `spirv-headers-devel` (Fedora / openSUSE). On Windows, the LunarG Vulkan SDK’s `Include` directory already contains these headers.
502
+
503
+ #### Common steps
504
+
505
+ Second, after verifying that you have followed all of the SDK installation/setup steps, use this command to make sure before proceeding:
506
+ ```bash
507
+ vulkaninfo
508
+ ```
509
+
510
+ Then, assuming you have `cd` into your llama.cpp folder and there are no errors with running `vulkaninfo`, you can proceed to build llama.cpp using the CMake commands below:
511
+ ```bash
512
+ cmake -B build -DGGML_VULKAN=1
513
+ cmake --build build --config Release
514
+ ```
515
+
516
+ Finally, after finishing your build, you should be able to do something like this:
517
+ ```bash
518
+ # Test the output binary
519
+ # "-ngl 99" should offload all of the layers to GPU for most (if not all) models.
520
+ ./build/bin/llama-cli -m "PATH_TO_MODEL" -p "Hi you how are you" -ngl 99
521
+
522
+ # You should see in the output, ggml_vulkan detected your GPU. For example:
523
+ # ggml_vulkan: Using Intel(R) Graphics (ADL GT2) | uma: 1 | fp16: 1 | warp size: 32
524
+ ```
525
+
526
+ ### For Mac users:
527
+
528
+ Generally, follow LunarG's [Getting Started with the MacOS Vulkan SDK](https://vulkan.lunarg.com/doc/sdk/latest/mac/getting_started.html) guide for installation and setup of the Vulkan SDK. There are two options of Vulkan drivers on macOS, both of which implement translation layers to map Vulkan to Metal. They can be hot-swapped by setting the `VK_ICD_FILENAMES` environment variable to point to the respective ICD JSON file.
529
+
530
+ Check the box for "KosmicKrisp" during the LunarG Vulkan SDK installation.
531
+
532
+ Set environment variable for the LunarG Vulkan SDK after installation (and optionally add to your shell profile for persistence):
533
+ ```bash
534
+ source /path/to/vulkan-sdk/setup-env.sh
535
+ ```
536
+
537
+ #### Using MoltenVK
538
+
539
+ MoltenVK is the default Vulkan driver installed with the LunarG Vulkan SDK on macOS, so you can use the above environment variable settings as is.
540
+
541
+ #### Using KosmicKrisp
542
+
543
+ Override the environment variable for KosmicKrisp:
544
+ ```bash
545
+ export VK_ICD_FILENAMES=$VULKAN_SDK/share/vulkan/icd.d/libkosmickrisp_icd.json
546
+ export VK_DRIVER_FILES=$VULKAN_SDK/share/vulkan/icd.d/libkosmickrisp_icd.json
547
+ ```
548
+
549
+ #### Build
550
+
551
+ This is the only step different from [above](#common-steps) instructions.
552
+ ```bash
553
+ cmake -B build -DGGML_VULKAN=1 -DGGML_METAL=OFF
554
+ cmake --build build --config Release
555
+ ```
556
+
557
+ ## CANN
558
+ This provides NPU acceleration using the AI cores of your Ascend NPU. And [CANN](https://www.hiascend.com/en/software/cann) is a hierarchical APIs to help you to quickly build AI applications and service based on Ascend NPU.
559
+
560
+ For more information about Ascend NPU in [Ascend Community](https://www.hiascend.com/en/).
561
+
562
+ Make sure to have the CANN toolkit installed. You can download it from here: [CANN Toolkit](https://www.hiascend.com/developer/download/community/result?module=cann)
563
+
564
+ Go to `llama.cpp` directory and build using CMake.
565
+ ```bash
566
+ cmake -B build -DGGML_CANN=on -DCMAKE_BUILD_TYPE=release
567
+ cmake --build build --config release
568
+ ```
569
+
570
+ You can test with:
571
+
572
+ ```bash
573
+ ./build/bin/llama-cli -m PATH_TO_MODEL -p "Building a website can be done in 10 steps:" -ngl 32
574
+ ```
575
+
576
+ If the following info is output on screen, you are using `llama.cpp` with the CANN backend:
577
+ ```bash
578
+ llm_load_tensors: CANN model buffer size = 13313.00 MiB
579
+ llama_new_context_with_model: CANN compute buffer size = 1260.81 MiB
580
+ ```
581
+
582
+ For detailed info, such as model/device supports, CANN install, please refer to [llama.cpp for CANN](./backend/CANN.md).
583
+
584
+ ## ZenDNN
585
+
586
+ ZenDNN provides optimized deep learning primitives for AMD EPYC™ CPUs. It accelerates matrix multiplication operations for inference workloads.
587
+
588
+ ### Compilation
589
+
590
+ - Using `CMake` on Linux (automatic build):
591
+
592
+ ```bash
593
+ cmake -B build -DGGML_ZENDNN=ON
594
+ cmake --build build --config Release
595
+ ```
596
+
597
+ The first build will automatically download and build ZenDNN, which may take 5-10 minutes. Subsequent builds will be much faster.
598
+
599
+ - Using `CMake` with custom ZenDNN installation:
600
+
601
+ ```bash
602
+ cmake -B build -DGGML_ZENDNN=ON -DZENDNN_ROOT=/path/to/zendnn/install
603
+ cmake --build build --config Release
604
+ ```
605
+
606
+ ### Testing
607
+
608
+ You can test with:
609
+
610
+ ```bash
611
+ ./build/bin/llama-cli -m PATH_TO_MODEL -p "Building a website can be done in 10 steps:" -n 50
612
+ ```
613
+
614
+ For detailed information about hardware support, setup instructions, and performance optimization, refer to [llama.cpp for ZenDNN](./backend/ZenDNN.md).
615
+
616
+ ## Arm® KleidiAI™
617
+ KleidiAI provides optimized Arm CPU microkernels used by the ggml CPU backend. Enabling it at build time makes those kernels available; it does not force every operation to use KleidiAI. At runtime, llama.cpp selects the best compatible CPU kernel from the detected CPU features, tensor type, operation shape, and active backend priority.
618
+
619
+ Supported targets:
620
+
621
+ | Platform | Supported ABI / architecture | Notes |
622
+ | --- | --- | --- |
623
+ | Linux | AArch64 / arm64 | Runtime CPU feature detection is automatic. |
624
+ | Android | `arm64-v8a` | Use the Android NDK command below for a portable build. |
625
+ | Apple | arm64 | Runtime CPU feature detection is automatic. Non-streaming SVE vector length is treated as unavailable. |
626
+ | Windows | arm64 | Runtime CPU feature detection is automatic. SMCU count is treated as unknown until a detection path is verified. |
627
+
628
+ `GGML_CPU_KLEIDIAI=ON` is valid only for AArch64/arm64 builds. Do not enable it for x86, 32-bit Arm, or Android ABIs other than `arm64-v8a`.
629
+
630
+ ### Native AArch64/arm64 build
631
+
632
+ From the llama.cpp source directory:
633
+
634
+ ```bash
635
+ cmake -S . -B build -DGGML_CPU_KLEIDIAI=ON
636
+ cmake --build build --config Release
637
+ ```
638
+
639
+ ### Android arm64-v8a NDK build
640
+
641
+ Set `ANDROID_NDK` to the Android NDK root, then run the following from the llama.cpp source directory. This command configures a portable Android `arm64-v8a` build with KleidiAI enabled and avoids Android dependencies that are not part of the NDK stable native API set.
642
+
643
+ ```bash
644
+ cmake -S . -B build-android \
645
+ -DCMAKE_BUILD_TYPE=Release \
646
+ -DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK/build/cmake/android.toolchain.cmake" \
647
+ -DANDROID_ABI=arm64-v8a \
648
+ -DANDROID_PLATFORM=android-28 \
649
+ -DGGML_CPU_KLEIDIAI=ON \
650
+ -DGGML_NATIVE=OFF \
651
+ -DGGML_OPENMP=OFF \
652
+ -DGGML_LLAMAFILE=OFF \
653
+ -DLLAMA_OPENSSL=OFF
654
+ cmake --build build-android --config Release --parallel
655
+ cmake --install build-android --prefix {install-dir} --config Release
656
+ ```
657
+
658
+ Important Android options:
659
+
660
+ - `GGML_CPU_KLEIDIAI=ON` enables KleidiAI for Android `arm64-v8a`.
661
+ - `GGML_NATIVE=OFF` is required for cross-compilation because the build host CPU is not the Android target CPU.
662
+ - `GGML_OPENMP=OFF` avoids adding an OpenMP runtime dependency to this NDK command-line build.
663
+ - `GGML_LLAMAFILE=OFF` avoids the llamafile backend, which is not supported on Android.
664
+ - `LLAMA_OPENSSL=OFF` avoids depending on OpenSSL, which is not part of the Android NDK stable native API set.
665
+
666
+ The Android Studio project under `examples/llama.android` enables KleidiAI automatically for `arm64-v8a`. For Android command-line CMake builds on `arm64-v8a`, pass `-DGGML_CPU_KLEIDIAI=ON` explicitly.
667
+
668
+ Global -march flags such as `-march=armv8.7a` flag are not required for a portable Android `arm64-v8a` build. Global `-march` flags raise the baseline instruction set for generic code. No manual architecture-specific source selection is required; llama.cpp selects compatible KleidiAI kernels at runtime. The KleidiAI libraries internal CMake handles the -march flags for each particular kernel.
669
+
670
+ ### Verifying the build
671
+
672
+ Run an installed or in-tree binary:
673
+
674
+ ```bash
675
+ ./build/bin/llama-cli -m PATH_TO_MODEL -p "What is a car?"
676
+ ```
677
+
678
+ If KleidiAI is enabled, the output contains a line similar to:
679
+
680
+ ```
681
+ load_tensors: CPU_KLEIDIAI model buffer size = 3474.00 MiB
682
+ ```
683
+
684
+ This confirms that the model has tensors allocated through the KleidiAI CPU buffer. It does not prove that every operation, or any specific SME-family operation, used a KleidiAI microkernel. Runtime CPU features, tensor type, operation shape, and backend priority still control dispatch.
685
+
686
+ Depending on the build target, another backend may have higher priority than the CPU backend. To force CPU execution for a run, disable higher priority backends at build time, for example `-DGGML_METAL=OFF`, or use a runtime device option such as `--device none` where supported.
687
+
688
+ ### Runtime dispatch
689
+
690
+ KleidiAI microkernels use Arm CPU features such as dotprod, i8mm, SVE, and SME/SME2. Build-time configuration makes the kernels available. Runtime dispatch selects a compatible kernel for the detected CPU and operation. Older or lower-feature CPUs fall back automatically to compatible kernels.
691
+
692
+ KleidiAI accelerates selected `GGML_OP_MUL_MAT` paths for F32 and common quantized formats. Exact coverage depends on the bundled KleidiAI version and the llama.cpp runtime selector, so unsupported tensor types, unsupported operation shapes, or higher priority backends may bypass KleidiAI even when the CPU supports the required Arm feature. This is also why a model may not use SME-family kernels on SME-capable hardware.
693
+
694
+ The current llama.cpp KleidiAI SVE selector only enables SVE kernels when the runtime SVE vector length is known to be QK8_0 bytes, currently 32 bytes. Linux and Android query this at runtime. Apple reports SVE capability separately from userspace non-streaming SVE availability, so llama.cpp treats the SVE vector length as unknown there. Windows exposes SVE feature presence but not the runtime SVE vector length used by this selector, so that value is also treated as unknown. Windows arm64 also treats SMCU count as unknown until a detection mechanism is verified.
695
+
696
+ The set of available SME-family kernels depends on the bundled KleidiAI version and the detected CPU capabilities. Production configuration does not require any KleidiAI runtime environment variables.
697
+
698
+ ### Diagnostics and debug overrides
699
+
700
+ KleidiAI runtime environment variables are diagnostics/debug overrides, not production configuration. Leave them unset for normal use.
701
+
702
+ `GGML_KLEIDIAI_SME` controls SME-family kernel selection and overrides the maximum number of threads assigned to selected quantized SME-family kernels:
703
+
704
+ - Not set: use automatic runtime detection.
705
+ - `0`: disable SME-family kernels.
706
+ - `<n> > 0`: enable compatible SME-family kernels and allow up to `<n>` threads for quantized SME-family kernels.
707
+
708
+ On Windows arm64, use `GGML_KLEIDIAI_SME=<n>` as the temporary diagnostics/debug override for SME thread-cap calibration until automatic SMCU count detection is verified.
709
+
710
+ If the CPU does not support the required SME-family capability for a bundled kernel, that kernel is disabled regardless of the environment variable.
711
+
712
+ ## OpenCL
713
+
714
+ This provides GPU acceleration through OpenCL on recent Adreno GPU.
715
+ More information about OpenCL backend can be found in [OPENCL.md](./backend/OPENCL.md) for more information.
716
+
717
+ ### Android
718
+
719
+ Assume NDK is available in `$ANDROID_NDK`. First, install OpenCL headers and ICD loader library if not available,
720
+
721
+ ```sh
722
+ mkdir -p ~/dev/llm
723
+ cd ~/dev/llm
724
+
725
+ git clone https://github.com/KhronosGroup/OpenCL-Headers && \
726
+ cd OpenCL-Headers && \
727
+ cp -r CL $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include
728
+
729
+ cd ~/dev/llm
730
+
731
+ git clone https://github.com/KhronosGroup/OpenCL-ICD-Loader && \
732
+ cd OpenCL-ICD-Loader && \
733
+ mkdir build_ndk && cd build_ndk && \
734
+ cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release \
735
+ -DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK/build/cmake/android.toolchain.cmake \
736
+ -DOPENCL_ICD_LOADER_HEADERS_DIR=$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include \
737
+ -DANDROID_ABI=arm64-v8a \
738
+ -DANDROID_PLATFORM=24 \
739
+ -DANDROID_STL=c++_shared && \
740
+ ninja && \
741
+ cp libOpenCL.so $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/aarch64-linux-android
742
+ ```
743
+
744
+ Then build llama.cpp with OpenCL enabled,
745
+
746
+ ```sh
747
+ cd ~/dev/llm
748
+
749
+ git clone https://github.com/ggml-org/llama.cpp && \
750
+ cd llama.cpp && \
751
+ mkdir build-android && cd build-android
752
+
753
+ cmake .. -G Ninja \
754
+ -DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK/build/cmake/android.toolchain.cmake \
755
+ -DANDROID_ABI=arm64-v8a \
756
+ -DANDROID_PLATFORM=android-28 \
757
+ -DBUILD_SHARED_LIBS=OFF \
758
+ -DGGML_OPENCL=ON
759
+
760
+ ninja
761
+ ```
762
+
763
+ ### Windows Arm64
764
+
765
+ First, install OpenCL headers and ICD loader library if not available,
766
+
767
+ ```powershell
768
+ mkdir -p ~/dev/llm
769
+
770
+ cd ~/dev/llm
771
+ git clone https://github.com/KhronosGroup/OpenCL-Headers && cd OpenCL-Headers
772
+ mkdir build && cd build
773
+ cmake .. -G Ninja `
774
+ -DBUILD_TESTING=OFF `
775
+ -DOPENCL_HEADERS_BUILD_TESTING=OFF `
776
+ -DOPENCL_HEADERS_BUILD_CXX_TESTS=OFF `
777
+ -DCMAKE_INSTALL_PREFIX="$HOME/dev/llm/opencl"
778
+ cmake --build . --target install
779
+
780
+ cd ~/dev/llm
781
+ git clone https://github.com/KhronosGroup/OpenCL-ICD-Loader && cd OpenCL-ICD-Loader
782
+ mkdir build && cd build
783
+ cmake .. -G Ninja `
784
+ -DCMAKE_BUILD_TYPE=Release `
785
+ -DCMAKE_PREFIX_PATH="$HOME/dev/llm/opencl" `
786
+ -DCMAKE_INSTALL_PREFIX="$HOME/dev/llm/opencl"
787
+ cmake --build . --target install
788
+ ```
789
+
790
+ Then build llama.cpp with OpenCL enabled,
791
+
792
+ ```powershell
793
+ cmake .. -G Ninja `
794
+ -DCMAKE_TOOLCHAIN_FILE="$HOME/dev/llm/llama.cpp/cmake/arm64-windows-llvm.cmake" `
795
+ -DCMAKE_BUILD_TYPE=Release `
796
+ -DCMAKE_PREFIX_PATH="$HOME/dev/llm/opencl" `
797
+ -DBUILD_SHARED_LIBS=OFF `
798
+ -DGGML_OPENCL=ON
799
+ ninja
800
+ ```
801
+
802
+ ## Android
803
+
804
+ To read documentation for how to build on Android, [click here](./android.md)
805
+
806
+ ## WebGPU
807
+
808
+ The WebGPU backend relies on [Dawn](https://dawn.googlesource.com/dawn). Follow the instructions [here](https://dawn.googlesource.com/dawn/+/refs/heads/main/docs/quickstart-cmake.md) to install Dawn locally so that llama.cpp can find it using CMake. The current implementation is up-to-date with Dawn commit `18eb229`.
809
+
810
+ In the llama.cpp directory, build with CMake:
811
+
812
+ ```
813
+ cmake -B build -DGGML_WEBGPU=ON
814
+ cmake --build build --config Release
815
+ ```
816
+
817
+ ### Browser Support
818
+
819
+ WebGPU allows cross-platform access to the GPU from supported browsers. We utilize [Emscripten](https://emscripten.org/) to compile ggml's WebGPU backend to WebAssembly. Emscripten does not officially support WebGPU bindings yet, but Dawn currently maintains its own WebGPU bindings called emdawnwebgpu.
820
+
821
+ Follow the instructions [here](https://dawn.googlesource.com/dawn/+/refs/heads/main/src/emdawnwebgpu/) to download or build the emdawnwebgpu package (Note that it might be safer to build the emdawnwebgpu package locally, so that it stays in sync with the version of Dawn you have installed above). When building using CMake, the path to the emdawnwebgpu port file needs to be set with the flag `EMDAWNWEBGPU_DIR`.
822
+
823
+ ## IBM Z & LinuxONE
824
+
825
+ To read documentation for how to build on IBM Z & LinuxONE, [click here](./build-s390x.md)
826
+
827
+ ## OpenVINO
828
+
829
+ [OpenVINO](https://docs.openvino.ai/) is an open-source toolkit for optimizing and deploying high-performance AI inference, specifically designed for Intel hardware (CPUs, GPUs, and NPUs).
830
+
831
+ For build instructions and usage examples, refer to [OPENVINO.md](backend/OPENVINO.md).
832
+
833
+ ### Hexagon
834
+
835
+ Check [README.md](./backend/snapdragon/README.md) for target specific build and run info.
836
+
837
+ ---
838
+ ## Notes about GPU-accelerated backends
839
+
840
+ The GPU may still be used to accelerate some parts of the computation even when using the `-ngl 0` option. You can fully disable GPU acceleration by using `--device none`.
841
+
842
+ In most cases, it is possible to build and use multiple backends at the same time. For example, you can build llama.cpp with both CUDA and Vulkan support by using the `-DGGML_CUDA=ON -DGGML_VULKAN=ON` options with CMake. At runtime, you can specify which backend devices to use with the `--device` option. To see a list of available devices, use the `--list-devices` option.
843
+
844
+ Backends can be built as dynamic libraries that can be loaded dynamically at runtime. This allows you to use the same llama.cpp binary on different machines with different GPUs. To enable this feature, use the `GGML_BACKEND_DL` option when building.
examples/llama.android/app/build.gradle.kts ADDED
@@ -0,0 +1,58 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ plugins {
2
+ alias(libs.plugins.android.application)
3
+ alias(libs.plugins.jetbrains.kotlin.android)
4
+ }
5
+
6
+ android {
7
+ namespace = "com.example.llama"
8
+ compileSdk = 36
9
+
10
+ defaultConfig {
11
+ applicationId = "com.example.llama.aichat"
12
+
13
+ minSdk = 33
14
+ targetSdk = 36
15
+
16
+ versionCode = 1
17
+ versionName = "1.0"
18
+
19
+ testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
20
+ vectorDrawables {
21
+ useSupportLibrary = true
22
+ }
23
+ }
24
+
25
+ buildTypes {
26
+ debug {
27
+ isMinifyEnabled = true
28
+ isShrinkResources = true
29
+ proguardFiles(
30
+ getDefaultProguardFile("proguard-android.txt"),
31
+ "proguard-rules.pro"
32
+ )
33
+ }
34
+ release {
35
+ isMinifyEnabled = true
36
+ isShrinkResources = true
37
+ proguardFiles(
38
+ getDefaultProguardFile("proguard-android-optimize.txt"),
39
+ "proguard-rules.pro"
40
+ )
41
+ }
42
+ }
43
+ compileOptions {
44
+ sourceCompatibility = JavaVersion.VERSION_17
45
+ targetCompatibility = JavaVersion.VERSION_17
46
+ }
47
+ }
48
+
49
+ dependencies {
50
+ implementation(libs.bundles.androidx)
51
+ implementation(libs.material)
52
+
53
+ implementation(project(":lib"))
54
+
55
+ testImplementation(libs.junit)
56
+ androidTestImplementation(libs.androidx.junit)
57
+ androidTestImplementation(libs.androidx.espresso.core)
58
+ }
examples/llama.android/build.gradle.kts ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ // Top-level build file where you can add configuration options common to all sub-projects/modules.
2
+ plugins {
3
+ alias(libs.plugins.android.application) apply false
4
+ alias(libs.plugins.android.library) apply false
5
+ alias(libs.plugins.jetbrains.kotlin.android) apply false
6
+ }
examples/llama.android/lib/build.gradle.kts ADDED
@@ -0,0 +1,79 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ plugins {
2
+ alias(libs.plugins.android.library)
3
+ alias(libs.plugins.jetbrains.kotlin.android)
4
+ }
5
+
6
+ android {
7
+ namespace = "com.arm.aichat"
8
+ compileSdk = 36
9
+
10
+ ndkVersion = "29.0.13113456"
11
+
12
+ defaultConfig {
13
+ minSdk = 33
14
+
15
+ testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
16
+ consumerProguardFiles("consumer-rules.pro")
17
+
18
+ ndk {
19
+ abiFilters += listOf("arm64-v8a", "x86_64")
20
+ }
21
+ externalNativeBuild {
22
+ cmake {
23
+ arguments += "-DCMAKE_BUILD_TYPE=Release"
24
+ arguments += "-DCMAKE_MESSAGE_LOG_LEVEL=DEBUG"
25
+ arguments += "-DCMAKE_VERBOSE_MAKEFILE=ON"
26
+
27
+ arguments += "-DBUILD_SHARED_LIBS=ON"
28
+ arguments += "-DLLAMA_BUILD_APP=OFF"
29
+ arguments += "-DLLAMA_BUILD_COMMON=ON"
30
+ arguments += "-DLLAMA_OPENSSL=OFF"
31
+
32
+ arguments += "-DGGML_NATIVE=OFF"
33
+ arguments += "-DGGML_BACKEND_DL=ON"
34
+ arguments += "-DGGML_CPU_ALL_VARIANTS=ON"
35
+ arguments += "-DGGML_LLAMAFILE=OFF"
36
+ }
37
+ }
38
+ aarMetadata {
39
+ minCompileSdk = 35
40
+ }
41
+ }
42
+ externalNativeBuild {
43
+ cmake {
44
+ path("src/main/cpp/CMakeLists.txt")
45
+ version = "3.31.6"
46
+ }
47
+ }
48
+ compileOptions {
49
+ sourceCompatibility = JavaVersion.VERSION_17
50
+ targetCompatibility = JavaVersion.VERSION_17
51
+ }
52
+ kotlin {
53
+ jvmToolchain(17)
54
+
55
+ compileOptions {
56
+ targetCompatibility = JavaVersion.VERSION_17
57
+ }
58
+ }
59
+
60
+ packaging {
61
+ resources {
62
+ excludes += "/META-INF/{AL2.0,LGPL2.1}"
63
+ }
64
+ }
65
+
66
+ publishing {
67
+ singleVariant("release") {
68
+ withJavadocJar()
69
+ }
70
+ }
71
+ }
72
+
73
+ dependencies {
74
+ implementation(libs.androidx.core.ktx)
75
+ implementation(libs.androidx.datastore.preferences)
76
+
77
+ testImplementation(libs.junit)
78
+ androidTestImplementation(libs.androidx.junit)
79
+ }
examples/sycl/build.sh ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+ # MIT license
3
+ # Copyright (C) 2024 Intel Corporation
4
+ # SPDX-License-Identifier: MIT
5
+
6
+ print_usage() {
7
+ echo "Usage: ./build.sh [fp32|fp16] [--help]"
8
+ echo ""
9
+ echo "Options:"
10
+ echo " fp32 Build with FP32 precision (default)"
11
+ echo " fp16 Build with FP16 precision (faster for long-prompt inference)"
12
+ echo " --help Print this help message"
13
+ }
14
+
15
+ PRECISION=fp32
16
+
17
+ for arg in "$@"; do
18
+ case "$arg" in
19
+ --help)
20
+ print_usage
21
+ exit 0
22
+ ;;
23
+ fp32|fp16)
24
+ PRECISION="$arg"
25
+ ;;
26
+ *)
27
+ echo "Error: unknown option '$arg'"
28
+ print_usage
29
+ exit 1
30
+ ;;
31
+ esac
32
+ done
33
+
34
+ mkdir -p build
35
+ cd build
36
+ source /opt/intel/oneapi/setvars.sh
37
+
38
+ if [ "$PRECISION" = "fp16" ]; then
39
+ #for FP16
40
+ cmake .. -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_SYCL_F16=ON -DLLAMA_OPENSSL=OFF # faster for long-prompt inference
41
+ else
42
+ #for FP32
43
+ cmake .. -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DLLAMA_OPENSSL=OFF
44
+ fi
45
+
46
+ #build example/main
47
+ #cmake --build . --config Release --target main
48
+
49
+ #build example/llama-bench
50
+ #cmake --build . --config Release --target llama-bench
51
+
52
+ #build all binary
53
+ cmake --build . --config Release -j$((($(nproc)+1)/2)) -v
examples/test-cmake/build-install.sh ADDED
@@ -0,0 +1,19 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+
3
+ set -e
4
+
5
+ rm -rf llama-build-install install
6
+
7
+ cmake --fresh -S ../../. -B llama-build-install -DCMAKE_BUILD_TYPE=Release \
8
+ -DBUILD_SHARED_LIBS=ON \
9
+ -DGGML_BACKEND_DL=ON \
10
+ -DGGML_CPU_ALL_VARIANTS=ON \
11
+ -DLLAMA_TESTS_INSTALL=OFF \
12
+ -DCMAKE_INSTALL_PREFIX="${PWD}/install" \
13
+ -DGGML_BACKEND_DIR="${PWD}/install/lib/llama.cpp" \
14
+ -DGGML_LIB_INSTALL_DIR="${PWD}/install/lib/llama.cpp" \
15
+ -DLLAMA_LIB_INSTALL_DIR="${PWD}/install/lib/llama.cpp" \
16
+ -DLLAMA_TOOLS_INSTALL=OFF
17
+
18
+ cmake --build llama-build-install --parallel 12
19
+ cmake --install llama-build-install
examples/test-cmake/build.sh ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ #!/bin/bash
2
+
3
+ set -e
4
+
5
+ cmake -S . -B build -DCMAKE_PREFIX_PATH="${PWD}/install"
6
+ cmake --build build
7
+ LD_LIBRARY_PATH="${PWD}/install/lib/llama.cpp:${PWD}/install/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" ./build/test-cmake
models/templates/Apertus-8B-Instruct.jinja ADDED
@@ -0,0 +1,327 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- macro render_typescript_type(param_spec, required_params, is_nullable=false) -%}
2
+ {%- if param_spec.type == "array" -%}
3
+ {%- if param_spec['items'] -%}
4
+ {%- if param_spec['items']['type'] == "string" -%}
5
+ {{- "string[]" }}
6
+ {%- elif param_spec['items']['type'] == "number" -%}
7
+ {{- "number[]" }}
8
+ {%- elif param_spec['items']['type'] == "integer" -%}
9
+ {{- "number[]" }}
10
+ {%- elif param_spec['items']['type'] == "boolean" -%}
11
+ {{- "boolean[]" }}
12
+ {%- else -%}
13
+ {%- set inner_type = render_typescript_type(param_spec['items'], required_params) -%}
14
+ {%- if inner_type == "object | object" or inner_type|length > 50 -%}
15
+ {{- "any[]" }}
16
+ {%- else -%}
17
+ {{- inner_type + "[]" }}
18
+ {%- endif -%}
19
+ {%- endif -%}
20
+ {%- if param_spec.nullable -%}
21
+ {{- " | null" }}
22
+ {%- endif -%}
23
+ {%- else -%}
24
+ {{- "any[]" }}
25
+ {%- if param_spec.nullable -%}
26
+ {{- " | null" }}
27
+ {%- endif -%}
28
+ {%- endif -%}
29
+ {%- elif param_spec.type is defined and param_spec.type is iterable and param_spec.type is not string and param_spec.type is not mapping and param_spec.type[0] is defined -%}
30
+ {#- Handle array of types like ["object", "object"] from Union[dict, list] #}
31
+ {%- if param_spec.type | length > 1 -%}
32
+ {{- param_spec.type | join(" | ") }}
33
+ {%- else -%}
34
+ {{- param_spec.type[0] }}
35
+ {%- endif -%}
36
+ {%- elif param_spec.oneOf -%}
37
+ {#- Handle oneOf schemas - check for complex unions and fallback to any #}
38
+ {%- set has_object_variants = false -%}
39
+ {%- for variant in param_spec.oneOf -%}
40
+ {%- if variant.type == "object" -%}
41
+ {%- set has_object_variants = true -%}
42
+ {%- endif -%}
43
+ {%- endfor -%}
44
+ {%- if has_object_variants and param_spec.oneOf|length > 1 -%}
45
+ {{- "any" }}
46
+ {%- else -%}
47
+ {%- for variant in param_spec.oneOf -%}
48
+ {{- render_typescript_type(variant, required_params) -}}
49
+ {%- if variant.description %}
50
+ {{- "// " + variant.description }}
51
+ {%- endif -%}
52
+ {%- if variant.default is defined %}
53
+ {{ "// default: " + variant.default|tojson }}
54
+ {%- endif -%}
55
+ {%- if not loop.last %}
56
+ {{- " | " }}
57
+ {% endif -%}
58
+ {%- endfor -%}
59
+ {%- endif -%}
60
+ {%- elif param_spec.type == "string" -%}
61
+ {%- if param_spec.enum -%}
62
+ {{- '"' + param_spec.enum|join('" | "') + '"' -}}
63
+ {%- else -%}
64
+ {{- "string" }}
65
+ {%- if param_spec.nullable %}
66
+ {{- " | null" }}
67
+ {%- endif -%}
68
+ {%- endif -%}
69
+ {%- elif param_spec.type == "number" -%}
70
+ {{- "number" }}
71
+ {%- elif param_spec.type == "integer" -%}
72
+ {{- "number" }}
73
+ {%- elif param_spec.type == "boolean" -%}
74
+ {{- "boolean" }}
75
+ {%- elif param_spec.type == "object" -%}
76
+ {%- if param_spec.properties -%}
77
+ {{- "{\n" }}
78
+ {%- for prop_name, prop_spec in param_spec.properties.items() -%}
79
+ {{- prop_name -}}
80
+ {%- if prop_name not in (param_spec.required or []) -%}
81
+ {{- "?" }}
82
+ {%- endif -%}
83
+ {{- ": " }}
84
+ {{ render_typescript_type(prop_spec, param_spec.required or []) }}
85
+ {%- if not loop.last -%}
86
+ {{-", " }}
87
+ {%- endif -%}
88
+ {%- endfor -%}
89
+ {{- "}" }}
90
+ {%- else -%}
91
+ {{- "object" }}
92
+ {%- endif -%}
93
+ {%- else -%}
94
+ {{- "any" }}
95
+ {%- endif -%}
96
+ {%- endmacro -%}
97
+
98
+ {%- macro render_tools(tools) -%}
99
+ {%- for tool in tools %}
100
+ {{- "// " + tool.function.description + "\n" }}
101
+ {{- "type "+ tool.function.name + " = " }}
102
+ {%- if tool.function.parameters and tool.function.parameters.properties %}
103
+ {{- "(_: {\n" }}
104
+ {%- for param_name, param_spec in tool.function.parameters.properties.items() %}
105
+ {%- if param_spec.description %}
106
+ {{- "// " + param_spec.description + "\n" }}
107
+ {%- endif %}
108
+ {{- param_name }}
109
+ {%- if param_name not in (tool.function.parameters.required or []) -%}
110
+ {{- "?" }}
111
+ {%- endif -%}
112
+ {{- ": " }}
113
+ {{- render_typescript_type(param_spec, tool.function.parameters.required or []) }}
114
+ {%- if param_spec.default is defined -%}
115
+ {%- if param_spec.enum %}
116
+ {{- ", // default: " + param_spec.default }}
117
+ {%- elif param_spec.oneOf %}
118
+ {{- "// default: " + param_spec.default }}
119
+ {%- else %}
120
+ {{- ", // default: " + param_spec.default|tojson }}
121
+ {%- endif -%}
122
+ {%- endif -%}
123
+ {%- if not loop.last %}
124
+ {{- ",\n" }}
125
+ {%- else %}
126
+ {{- "\n" }}
127
+ {%- endif -%}
128
+ {%- endfor %}
129
+ {{- "}) => any;" }}
130
+ {%- else -%}
131
+ {{- "() => any;" }}
132
+ {%- endif -%}
133
+ {%- if not loop.last -%}
134
+ {{- "\n" }}
135
+ {%- endif -%}
136
+ {%- endfor %}
137
+ {%- endmacro -%}
138
+
139
+ {{ bos_token }}
140
+
141
+ {%- set system_token = '<|system_start|>' -%}
142
+ {%- set end_system_token = '<|system_end|>' -%}
143
+ {%- set developer_token = '<|developer_start|>' -%}
144
+ {%- set end_developer_token = '<|developer_end|>' -%}
145
+ {%- set user_token = '<|user_start|>' -%}
146
+ {%- set end_user_token = '<|user_end|>' -%}
147
+ {%- set assistant_token = '<|assistant_start|>' -%}
148
+ {%- set end_assistant_token = '<|assistant_end|>' -%}
149
+ {%- set inner_token = '<|inner_prefix|>' -%}
150
+ {%- set outer_token = '<|inner_suffix|>' -%}
151
+ {%- set tool_calls_token = '<|tools_prefix|>' -%}
152
+ {%- set end_tool_calls_token = '<|tools_suffix|>' -%}
153
+
154
+ {%- set ns = namespace(in_assistant=false, in_tool=false, in_inner=false, assistant_format=none) -%}
155
+
156
+ {%- if messages and messages[0].role == 'system' -%}
157
+ {%- if "content" in messages[0] -%}
158
+ {%- if messages[0].content is string -%}
159
+ {{ system_token + messages[0].content + end_system_token }}
160
+ {%- elif messages[0].content is mapping and "text" in messages[0].content -%}
161
+ {{ system_token + messages[0].content.text + end_system_token }}
162
+ {%- else -%}
163
+ {{- raise_exception("Invalid system message") -}}
164
+ {%- endif -%}
165
+ {%- else -%}
166
+ {{- raise_exception("Invalid system message") -}}
167
+ {%- endif -%}
168
+ {%- set loop_messages = messages[1:] -%}
169
+ {%- else -%}
170
+ {{ system_token + 'You are Apertus, a helpful assistant created by the SwissAI initiative.\nKnowledge cutoff: 2024-04\nCurrent date: ' + strftime_now('%Y-%m-%d') + end_system_token }}
171
+ {%- set loop_messages = messages -%}
172
+ {%- endif -%}
173
+
174
+ {{ developer_token + 'Deliberation: ' }}
175
+ {%- if enable_thinking is defined and enable_thinking -%}
176
+ {{ 'enabled\n' }}
177
+ {%- else -%}
178
+ {{ 'disabled\n' }}
179
+ {%- endif -%}
180
+ {%- if tools is defined and tools -%}
181
+ {{ 'Tool Capabilities:\n' + render_tools(tools) }}
182
+ {%- else -%}
183
+ {{ 'Tool Capabilities: disabled' }}
184
+ {%- endif -%}
185
+ {{ end_developer_token }}
186
+
187
+ {%- for message in loop_messages -%}
188
+ {%- if message.role == 'user' -%}
189
+ {%- set ns.in_inner = false -%}
190
+ {%- if ns.in_tool -%}
191
+ {{ ']' }}
192
+ {%- set ns.in_tool = false -%}
193
+ {%- endif -%}
194
+ {%- if ns.in_assistant -%}
195
+ {{ end_assistant_token }}
196
+ {%- set ns.in_assistant = false -%}
197
+ {%- endif -%}
198
+ {%- if "content" in message -%}
199
+ {{ user_token }}
200
+ {%- if message.content is string -%}
201
+ {{ message.content }}
202
+ {%- elif message.content is mapping and "parts" in message.content -%}
203
+ {%- set parts = message.content.parts -%}
204
+ {%- for part in parts -%}
205
+ {%- if part.type == "text" -%}
206
+ {{ part.text }}
207
+ {%- else -%}
208
+ {{- raise_exception("Invalid user part: " + part.type) -}}
209
+ {%- endif -%}
210
+ {%- endfor -%}
211
+ {%- else -%}
212
+ {{- raise_exception("Invalid user message: " + message.role) -}}
213
+ {%- endif -%}
214
+ {{ end_user_token }}
215
+ {%- endif -%}
216
+ {%- elif message.role == 'assistant' -%}
217
+ {%- if not ns.in_assistant -%}
218
+ {{ assistant_token }}
219
+ {%- set ns.in_assistant = true -%}
220
+ {%- endif -%}
221
+ {%- if "content" in message and message.content is not none -%}
222
+ {%- if message.content is string and (ns.assistant_format is none or ns.assistant_format == "string") -%}
223
+ {%- if ns.in_tool -%}
224
+ {{ ']' }}
225
+ {%- set ns.in_tool = false -%}
226
+ {%- endif -%}
227
+ {%- set ns.assistant_format = "string" -%}
228
+ {{ message.content }}
229
+ {%- elif message.content is mapping and "blocks" in message.content and (ns.assistant_format is none or ns.assistant_format == "mapping") -%}
230
+ {%- set ns.assistant_format = "mapping" -%}
231
+ {%- set blocks = message.content.blocks -%}
232
+ {%- for block in blocks -%}
233
+ {%- if block.type == 'thoughts' -%}
234
+ {%- if ns.in_tool -%}
235
+ {{ ']' }}
236
+ {%- set ns.in_tool = false -%}
237
+ {%- endif -%}
238
+ {%- if not ns.in_inner -%}
239
+ {%- set ns.in_inner = true -%}
240
+ {{ inner_token }}
241
+ {%- endif -%}
242
+ {{ block.text }}
243
+ {%- elif block.type == 'tool_calls' -%}
244
+ {%- if ns.in_tool -%}
245
+ {{ ']' }}
246
+ {%- set ns.in_tool = false -%}
247
+ {%- endif -%}
248
+ {%- if ns.in_inner and not loop.first and block.calls|length == 1 and block.calls[0].name == 'display_answers' -%}
249
+ {%- set ns.in_inner = false -%}
250
+ {{ outer_token }}
251
+ {%- endif -%}
252
+ {{ tool_calls_token + '[' }}
253
+ {%- for tool_call in block.calls -%}
254
+ {{- '{"' + tool_call.name + '": ' + tool_call.arguments + '}' }}
255
+ {%- if not loop.last -%}
256
+ {{- ", " }}
257
+ {%- endif -%}
258
+ {%- endfor -%}
259
+ {{ ']' + end_tool_calls_token }}
260
+ {%- elif block.type == 'tool_outputs' -%}
261
+ {%- if ns.in_tool -%}
262
+ {{- raise_exception("Cannot have both tool outputs as separate messages and tool outputs as blocks") -}}
263
+ {%- endif -%}
264
+ {{ '[' }}
265
+ {%- for tool_output in block.outputs -%}
266
+ {{- tool_output.output }}
267
+ {%- if not loop.last -%}
268
+ {{- ", " }}
269
+ {%- endif -%}
270
+ {%- endfor -%}
271
+ {{- ']' }}
272
+ {%- elif block.type == 'response' -%}
273
+ {%- if ns.in_tool -%}
274
+ {{ ']' }}
275
+ {%- set ns.in_tool = false -%}
276
+ {%- endif -%}
277
+ {%- if (not loop.first and ns.in_inner) or (ns.in_assistant and ns.in_inner) -%}
278
+ {%- set ns.in_inner = false -%}
279
+ {{ outer_token }}
280
+ {%- endif -%}
281
+ {{ block.text }}
282
+ {%- else -%}
283
+ {{- raise_exception("Invalid assistant block type: " + block.type) -}}
284
+ {%- endif -%}
285
+ {%- endfor -%}
286
+ {%- else -%}
287
+ {{- raise_exception("Invalid assistant content '" + message.content + "', expected " + ns.assistant_format) -}}
288
+ {%- endif -%}
289
+ {%- elif "tool_calls" not in message -%}
290
+ {{- raise_exception("Invalid assistant message " + message) -}}
291
+ {%- endif -%}
292
+ {%- if "tool_calls" in message and message.tool_calls -%}
293
+ {{ tool_calls_token + '[' }}
294
+ {%- for tool_call in message.tool_calls -%}
295
+ {%- if tool_call.type == 'function' -%}
296
+ {%- set function = tool_call.function -%}
297
+ {{- '{"' + function.name + '": ' + function.arguments|tojson + '}' }}
298
+ {%- if not loop.last -%}
299
+ {{- ", " }}
300
+ {%- endif -%}
301
+ {%- else -%}
302
+ {{- raise_exception("Invalid tool call type: " + tool_call.type) -}}
303
+ {%- endif -%}
304
+ {%- endfor -%}
305
+ {{ ']' + end_tool_calls_token }}
306
+ {%- endif -%}
307
+ {%- elif message.role == 'tool' -%}
308
+ {%- if not ns.in_assistant -%}
309
+ {{- raise_exception("Tool message outside of assistant") -}}
310
+ {%- endif -%}
311
+ {%- if not ns.in_tool -%}
312
+ {{ '[' }}
313
+ {%- set ns.in_tool = true -%}
314
+ {%- else -%}
315
+ {{ ", "}}
316
+ {%- endif -%}
317
+ {{ message.content }}
318
+ {%- else -%}
319
+ {{- raise_exception("Invalid message role") -}}
320
+ {%- endif -%}
321
+ {%- endfor -%}
322
+ {%- if ns.in_tool -%}
323
+ {{ ']' }}
324
+ {%- endif -%}
325
+ {%- if add_generation_prompt -%}
326
+ {{ assistant_token }}
327
+ {%- endif -%}
models/templates/Apriel-1.6-15b-Thinker-fixed.jinja ADDED
@@ -0,0 +1,171 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {# ---------------------------------------------------------------------- #}
2
+ {# ƛƬ Default setup and flags #}
3
+ {# ---------------------------------------------------------------------- #}
4
+ {%- set messages = messages or [] -%}
5
+ {%- set tools = tools or [] -%}
6
+ {%- set add_generation_prompt = add_generation_prompt or false -%}
7
+ {%- set available_tool_string = '' -%}
8
+ {%- set add_tool_id = true -%}
9
+ {%- set add_thoughts = true -%} {# whether to include <thinking> reasoning blocks #}
10
+ {# Optional token placeholders (safe defaults) #}
11
+ {%- set bos_token = bos_token or '' -%}
12
+ {%- set eos_token = eos_token or '' -%}
13
+ {# ---------------------------------------------------------------------- #}
14
+ {# Core reasoning prompt and assistant reasoning prefix #}
15
+ {# ---------------------------------------------------------------------- #}
16
+ {%- set reasoning_prompt -%}
17
+ You are a thoughtful, systematic AI assistant from ServiceNow Language Models (SLAM) lab.
18
+ Analyze each question carefully, present your reasoning step-by-step, then provide the final
19
+ response after the marker [BEGIN FINAL RESPONSE].
20
+ {%- endset -%}
21
+ {%- set reasoning_asst_turn_start = 'Here are my reasoning steps:\n' -%}
22
+ {# ---------------------------------------------------------------------- #}
23
+ {# Tool list and tool call output format #}
24
+ {# ---------------------------------------------------------------------- #}
25
+ {%- if tools|length > 0 -%}
26
+ {%- set available_tool_string -%}
27
+ You are provided with function signatures within <available_tools></available_tools> XML tags.
28
+ You may call one or more functions to assist with the user query.
29
+ Don't make assumptions about the arguments. You should infer the argument values from previous
30
+ user responses and the system message.
31
+ Here are the available tools:
32
+ <available_tools>
33
+ {% for tool in tools %}{{ tool|string }}{% endfor %}
34
+
35
+ </available_tools>.
36
+
37
+ Return all function calls as a list of JSON objects within <tool_calls></tool_calls> XML tags.
38
+ Each JSON object should contain a function name and arguments as follows:
39
+ <tool_calls>[
40
+ {"name": <function-name-1>, "arguments": <args-dict-1>},
41
+ {"name": <function-name-2>, "arguments": <args-dict-2>},
42
+ ...
43
+ ]</tool_calls>
44
+ {%- endset -%}
45
+ {%- endif -%}
46
+ {# ---------------------------------------------------------------------- #}
47
+ {# Start system block if first message is not system #}
48
+ {# ---------------------------------------------------------------------- #}
49
+ {%- if messages|length > 0 and messages[0]['role'] != 'system' -%}
50
+ {%- if tools|length > 0 -%}
51
+ {{ bos_token + '<|begin_system|>\n' + reasoning_prompt + '\n' + available_tool_string + '\n' }}
52
+ {%- else -%}
53
+ {{ bos_token + '<|begin_system|>\n' + reasoning_prompt + '\n' }}
54
+ {%- endif -%}
55
+ {%- endif -%}
56
+ {# ---------------------------------------------------------------------- #}
57
+ {# Iterate through messages #}
58
+ {# ---------------------------------------------------------------------- #}
59
+ {%- for message in messages -%}
60
+
61
+ {# ---------------- USER MESSAGE ---------------- #}
62
+ {%- if message['role'] == 'user' -%}
63
+ {{ '<|begin_user|>\n' }}
64
+ {%- if message['content'] is not string -%}
65
+ {%- for chunk in message['content'] -%}
66
+ {%- if chunk['type'] == 'text' -%}
67
+ {{ chunk['text'] }}
68
+ {%- elif chunk['type'] in ['image', 'image_url'] -%}
69
+ {{ '[IMG]' }}
70
+ {%- else -%}
71
+ {{ raise_exception('Unrecognized content type!') }}
72
+ {%- endif -%}
73
+ {%- endfor -%}
74
+ {%- else -%}
75
+ {{ message['content'] }}
76
+ {%- endif -%}
77
+
78
+ {# ---------------- SYSTEM MESSAGE ---------------- #}
79
+ {%- elif message['role'] == 'system' -%}
80
+ {%- set sys_content = message.get('content', '') -%}
81
+ {%- if sys_content and sys_content|length > 0 -%}
82
+ {%- if sys_content is string -%}
83
+ {%- set system_message = sys_content -%}
84
+ {%- else -%}
85
+ {%- set system_message = sys_content[0]['text'] -%}
86
+ {%- endif -%}
87
+ {%- else -%}
88
+ {%- set system_message = '' -%}
89
+ {%- endif -%}
90
+
91
+ {%- if tools|length > 0 -%}
92
+ {{ bos_token + '<|begin_system|>\n' + reasoning_prompt + '\n' + system_message + '\n' + available_tool_string + '\n' }}
93
+ {%- else -%}
94
+ {{ bos_token + '<|begin_system|>\n' + reasoning_prompt + '\n' + system_message + '\n' }}
95
+ {%- endif -%}
96
+
97
+ {# ---------------- ASSISTANT MESSAGE ---------------- #}
98
+ {%- elif message['role'] == 'assistant' -%}
99
+ {%- if loop.last -%}
100
+ {%- set add_tool_id = false -%}
101
+ {%- endif -%}
102
+
103
+ {{ '\n<|begin_assistant|>\n' }}
104
+
105
+ {%- if add_thoughts and message.get('reasoning_content') and loop.last -%}
106
+ {{ message['reasoning_content'] + '\n[BEGIN FINAL RESPONSE]\n' }}
107
+ {%- endif -%}
108
+
109
+ {%- set asst_content = message.get('content', '') -%}
110
+ {%- if asst_content and asst_content|length > 0 -%}
111
+ {%- if asst_content is not string -%}
112
+ {%- set asst_text = asst_content[0]['text'] -%}
113
+ {%- else -%}
114
+ {%- set asst_text = asst_content -%}
115
+ {%- endif -%}
116
+ {# For historical turns (not the last), strip reasoning and keep only final response #}
117
+ {%- if not loop.last and '[BEGIN FINAL RESPONSE]' in asst_text -%}
118
+ {{- asst_text.split('[BEGIN FINAL RESPONSE]')[-1] | trim -}}
119
+ {%- else -%}
120
+ {{- asst_text -}}
121
+ {%- endif -%}
122
+ {%- elif message.get('chosen') and message['chosen']|length > 0 -%}
123
+ {{ message['chosen'][0] }}
124
+ {%- endif -%}
125
+
126
+ {# Tool call output #}
127
+ {%- set tool_calls = message.get('tool_calls', []) -%}
128
+ {%- if tool_calls and tool_calls|length > 0 -%}
129
+ {{ '\n<tool_calls>[' }}
130
+ {%- for tool_call in tool_calls -%}
131
+ {{ '{"name": "' + tool_call['function']['name'] + '", "arguments": ' + tool_call['function']['arguments']|tojson }}
132
+ {%- if add_tool_id == true and 'id' in tool_call -%}
133
+ {{ ', "id": "' + tool_call['id'] + '"' }}
134
+ {%- endif -%}
135
+ {{ '}' }}
136
+ {%- if not loop.last -%}{{ ', ' }}{%- endif -%}
137
+ {%- endfor -%}
138
+ {{ ']</tool_calls>' }}
139
+ {%- endif -%}
140
+
141
+ {%- set training_prompt = training_prompt if (training_prompt is defined) else false -%}
142
+ {%- if not loop.last or training_prompt -%}
143
+ {{ '\n<|end|>\n' }}
144
+ {%- endif -%}
145
+
146
+ {# ---------------- TOOL RESULT MESSAGE ---------------- #}
147
+ {%- elif message['role'] == 'tool' -%}
148
+ {%- set tool_content = message.get('content', '') -%}
149
+ {%- if tool_content is string -%}
150
+ {%- set tool_message = tool_content -%}
151
+ {%- else -%}
152
+ {%- set tool_message = tool_content[0]['text'] if tool_content else '' -%}
153
+ {%- endif -%}
154
+ {{ '<|begin_tool_result|>\n' + tool_message|string + '\n' }}
155
+
156
+ {# ---------------- CONTENT MESSAGE ---------------- #}
157
+ {%- elif message['role'] == 'content' -%}
158
+ {%- set msg_content = message.get('content', '') -%}
159
+ {%- if msg_content is not string -%}
160
+ {{ '<|begin_content|>\n' + msg_content[0]['text'] + '\n' }}
161
+ {%- else -%}
162
+ {{ '<|begin_content|>\n' + msg_content + '\n' }}
163
+ {%- endif -%}
164
+ {%- endif -%}
165
+
166
+ {# ---------------- REASONING PROMPT BEFORE NEXT ASSISTANT ---------------- #}
167
+ {%- if loop.last and add_generation_prompt and message['role'] != 'assistant' -%}
168
+ {{ '\n<|begin_assistant|>\n' + reasoning_asst_turn_start }}
169
+ {%- endif -%}
170
+
171
+ {%- endfor -%}
models/templates/Bielik-11B-v3.0-Instruct.jinja ADDED
@@ -0,0 +1,77 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {{ bos_token }}
2
+ {%- if messages[0]['role'] == 'system' %}
3
+ {%- set system_message = messages[0]['content'] %}
4
+ {%- set loop_start_index = 1 %}
5
+ {%- else %}
6
+ {%- set system_message = "" %}
7
+ {%- set loop_start_index = 0 %}
8
+ {%- endif %}
9
+
10
+ {%- if system_message or tools %}
11
+ {{- '<|im_start|>system\n' }}
12
+ {%- if system_message %}
13
+ {{- system_message }}
14
+ {%- endif %}
15
+ {%- if tools %}
16
+ {{- '\n\nMasz dostęp do następujących narzędzi. Definicje narzędzi znajdują się poniżej wewnątrz znaczników <|function_list|>:\n<|function_list|>\n' }}
17
+ {{- '[' }}
18
+ {%- for tool in tools %}
19
+ {{- tool | tojson }}
20
+ {%- if not loop.last %}
21
+ {{- ',\n' }}
22
+ {%- endif %}
23
+ {%- endfor %}
24
+ {{- ']\n<|function_list|>\n\nAby wywołać narzędzie, użyj formatu <tool_call>{"name": "nazwa_narzędzia", "arguments": {"argument": "wartość"}}</tool_call>. Wyniki działania narzędzi zostaną przekazane z markerem <|function_output|>.\n' }}
25
+ {%- endif %}
26
+ {%- if enable_thinking %}
27
+ {{- '\n\nZanim odpowiesz na pytanie, najpierw przemyśl swoje kroki i umieść swoje myśli wewnątrz tagów <think>...</think>. Musisz najpierw pomyśleć, zanim udzielisz odpowiedzi. ' -}}
28
+ {{- 'WAŻNE: Powinieneś myśleć w tym samym języku, co pytanie użytkownika. Jeśli pytanie jest zadane po polsku, powinieneś również myśleć po polsku. Jeśli pytanie jest po angielsku, myślisz również po angielsku itd. ' }}
29
+ {{- '** PAMIĘTAJ! ** Pytanie po polsku -> myślenie po polsku -> odpowiedź po polsku!' -}}
30
+ {{- '*** BARDZO WAŻNE!!! *** Jesteś Bielikiem, polskim modelem językowym. Twoją główną cechą jest umiejętność pisania po polsku. Jeśli użytkownik zadaje Ci pytania po polsku, ZAWSZE odpowiadaj po polsku. ' -}}
31
+ {{- 'Nawet, jeśli korzystasz z narzędzia, którego większość instrukcji jest po angielsku, powinieneś przede wszystkim odpowiadać po polsku, jeśli użytkownik zadaje pytanie w tym języku. ' -}}
32
+ {%- endif %}
33
+ {{- '<|im_end|>\n' }}
34
+ {%- endif %}
35
+
36
+ {%- for message in messages[loop_start_index:] %}
37
+ {%- if message['role'] == 'user' %}
38
+ {{- '<|im_start|>user\n' + message['content'] + '<|im_end|>\n' }}
39
+ {%- elif message['role'] == 'assistant' %}
40
+ {{- '<|im_start|>assistant\n' }}
41
+ {%- set content = message.content | default('') %}
42
+ {%- set reasoning_content = message.reasoning_content | default('') %}
43
+ {%- if not reasoning_content and '<think>' in content and '</think>' in content %}
44
+ {%- set reasoning_parts = content.split('</think>') %}
45
+ {%- set reasoning_content = reasoning_parts[0].split('<think>')[-1] %}
46
+ {%- set content = reasoning_parts[1:] | join('</think>') %}
47
+ {%- endif %}
48
+ {%- if reasoning_content %}
49
+ {{- '<think>\n' + reasoning_content.strip() + '\n</think>\n' }}
50
+ {%- endif %}
51
+ {{- content.lstrip() }}
52
+ {%- if message.tool_calls %}
53
+ {%- for tool_call in message.tool_calls %}
54
+ {%- if tool_call.function %}
55
+ {%- set tool_call = tool_call.function %}
56
+ {%- endif %}
57
+ {{- '\n<tool_call>\n{"name": "' + tool_call.name + '", "arguments": ' + (tool_call.arguments if tool_call.arguments is string else tool_call.arguments | tojson) + '}\n</tool_call>' }}
58
+ {%- endfor %}
59
+ {%- endif %}
60
+ {{- '<|im_end|>\n' }}
61
+ {%- elif message['role'] == 'tool' %}
62
+ {%- if loop.index0 == 0 or messages[loop.index0 - 1]['role'] != 'tool' %}
63
+ {{- '<|im_start|>user\n' }}
64
+ {%- endif %}
65
+ {{- '<|function_output|>' + message['content'] }}
66
+ {%- if loop.last or messages[loop.index0 + 1]['role'] != 'tool' %}
67
+ {{- '<|im_end|>\n' }}
68
+ {%- endif %}
69
+ {%- endif %}
70
+ {%- endfor %}
71
+
72
+ {%- if add_generation_prompt %}
73
+ {{- '<|im_start|>assistant\n' }}
74
+ {%- if enable_thinking %}
75
+ {{- '<think>\n' }}
76
+ {%- endif %}
77
+ {%- endif %}
models/templates/ByteDance-Seed-OSS.jinja ADDED
@@ -0,0 +1,171 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {# ----------‑‑‑ special token variables ‑‑‑---------- #}
2
+ {%- set bos_token = '<seed:bos>' -%}
3
+ {%- set eos_token = '<seed:eos>' -%}
4
+ {%- set pad_token = '<seed:pad>' -%}
5
+ {%- set toolcall_begin_token = '<seed:tool_call>' -%}
6
+ {%- set toolcall_end_token = '</seed:tool_call>' -%}
7
+ {%- set think_begin_token = '<seed:think>' -%}
8
+ {%- set think_end_token = '</seed:think>' -%}
9
+ {%- set budget_begin_token = '<seed:cot_budget_reflect>'-%}
10
+ {%- set budget_end_token = '</seed:cot_budget_reflect>'-%}
11
+ {# -------------- reflection-interval lookup -------------- #}
12
+ {%- if not thinking_budget is defined %}
13
+ {%- set thinking_budget = -1 -%}
14
+ {%- endif -%}
15
+ {%- set budget_reflections_v05 = {
16
+ 0: 0,
17
+ 512: 128,
18
+ 1024: 256,
19
+ 2048: 512,
20
+ 4096: 512,
21
+ 8192: 1024,
22
+ 16384: 1024
23
+ } -%}
24
+ {# Find the first gear that is greater than or equal to the thinking_budget. #}
25
+ {%- set ns = namespace(interval = None) -%}
26
+ {%- for k, v in budget_reflections_v05 | dictsort -%}
27
+ {%- if ns.interval is none and thinking_budget <= k -%}
28
+ {%- set ns.interval = v -%}
29
+ {%- endif -%}
30
+ {%- endfor -%}
31
+ {# If it exceeds the maximum gear, use the value of the last gear #}
32
+ {%- if ns.interval is none -%}
33
+ {%- set ns.interval = budget_reflections_v05[16384] -%}
34
+ {%- endif -%}
35
+ {# ---------- Preprocess the system message ---------- #}
36
+ {%- if messages[0]["role"] == "system" %}
37
+ {%- set system_message = messages[0]["content"] %}
38
+ {%- set loop_messages = messages[1:] %}
39
+ {%- else %}
40
+ {%- set loop_messages = messages %}
41
+ {%- endif %}
42
+ {# ---------- Ensure tools exist ---------- #}
43
+ {%- if not tools is defined or tools is none %}
44
+ {%- set tools = [] %}
45
+ {%- endif %}
46
+ {# tools2doc.jinja #}
47
+ {%- macro py_type(t) -%}
48
+ {%- if t == "string" -%}str
49
+ {%- elif t in ("number", "integer") -%}int
50
+ {%- elif t == "boolean" -%}bool
51
+ {%- elif t == "array" -%}list
52
+ {%- else -%}Any{%- endif -%}
53
+ {%- endmacro -%}
54
+ {# ---------- Output the system block ---------- #}
55
+ {%- if system_message is defined %}
56
+ {{ bos_token + "system\n" + system_message }}
57
+ {%- else %}
58
+ {%- if tools is iterable and tools | length > 0 %}
59
+ {{ bos_token + "system\nYou are Doubao, a helpful AI assistant. You may call one or more functions to assist with the user query." }}
60
+ {%- endif %}
61
+ {%- endif %}
62
+ {%- if use_json_tooldef is defined and use_json_tooldef %}
63
+
64
+ {{"Tool List:\nYou are authorized to use the following tools (described in JSON Schema format). Before performing any task, you must decide how to call them based on the descriptions and parameters of these tools."}}
65
+ {{ tools | tojson(ensure_ascii=False) }}
66
+ {%- else %}
67
+ {%- for item in tools if item.type == "function" %}
68
+
69
+
70
+ Function:
71
+ def {{ item.function.name }}(
72
+ {%- for name, spec in item.function.parameters.properties.items() %}
73
+ {{- name }}: {{ py_type(spec.type) }}{% if not loop.last %},{% endif %}
74
+ {%- endfor %}):
75
+ """
76
+ {{ item.function.description | trim }}
77
+
78
+ {# ---------- Args ---------- #}
79
+ {%- if item.function.parameters.properties %}
80
+ Args:
81
+ {%- for name, spec in item.function.parameters.properties.items() %}
82
+
83
+ - {{ name }} ({{ py_type(spec.type) }})
84
+ {%- if name in item.function.parameters.required %} [必填]{% else %} [选填]{% endif %}:
85
+ {{- " " ~ (spec.description or "") }}
86
+ {%- endfor %}
87
+ {%- endif %}
88
+
89
+ {# ---------- Returns ---------- #}
90
+ {%- if item.function.returns is defined
91
+ and item.function.returns.properties is defined
92
+ and item.function.returns.properties %}
93
+ Returns:
94
+ {%- for name, spec in item.function.returns.properties.items() %}
95
+
96
+ - {{ name }} ({{ py_type(spec.type) }}):
97
+ {{- " " ~ (spec.description or "") }}
98
+ {%- endfor %}
99
+ {%- endif %}
100
+
101
+ """
102
+ {%- endfor %}
103
+ {%- endif %}
104
+ {%- if tools is iterable and tools | length > 0 %}
105
+
106
+ {{"工具调用请遵循如下格式:\n<seed:tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>value_1</parameter>\n<parameter=example_parameter_2>This is the value for the second parameter\nthat can span\nmultiple lines</parameter>\n</function>\n</seed:tool_call>\n"}}
107
+ {%- endif %}
108
+ {# End the system block line #}
109
+ {%- if system_message is defined or tools is iterable and tools | length > 0 %}
110
+ {{ eos_token }}
111
+ {%- endif %}
112
+ {# ---------- Thinking Budget ---------- #}
113
+ {%- if thinking_budget is defined %}
114
+ {%- if thinking_budget == 0 %}
115
+ {{ bos_token+"system" }}
116
+ {{ "You are an intelligent assistant that can answer questions in one step without the need for reasoning and thinking, that is, your thinking budget is 0. Next, please skip the thinking process and directly start answering the user's questions." }}
117
+ {{ eos_token }}
118
+ {%- elif not thinking_budget == -1 %}
119
+ {{ bos_token+"system" }}
120
+ {{ "You are an intelligent assistant with reflective ability. In the process of thinking and reasoning, you need to strictly follow the thinking budget, which is "}}{{thinking_budget}}{{". That is, you need to complete your thinking within "}}{{thinking_budget}}{{" tokens and start answering the user's questions. You will reflect on your thinking process every "}}{{ns.interval}}{{" tokens, stating how many tokens have been used and how many are left."}}
121
+ {{ eos_token }}
122
+ {%- endif %}
123
+ {%- endif %}
124
+ {# ---------- List the historical messages one by one ---------- #}
125
+ {%- for message in loop_messages %}
126
+ {%- if message.role == "assistant"
127
+ and message.tool_calls is defined
128
+ and message.tool_calls is iterable
129
+ and message.tool_calls | length > 0 %}
130
+ {{ bos_token + message.role }}
131
+ {%- if message.reasoning_content is defined and message.reasoning_content is string and message.reasoning_content | trim | length > 0 %}
132
+ {{ "\n" + think_begin_token + message.reasoning_content | trim + think_end_token }}
133
+ {%- endif %}
134
+ {%- if message.content is defined and message.content is string and message.content | trim | length > 0 %}
135
+ {{ "\n" + message.content | trim + "\n" }}
136
+ {%- endif %}
137
+ {%- for tool_call in message.tool_calls %}
138
+ {%- if tool_call.function is defined %}{% set tool_call = tool_call.function %}{% endif %}
139
+ {{ "\n" + toolcall_begin_token + "\n<function=" + tool_call.name + ">\n" }}
140
+ {%- if tool_call.arguments is defined %}
141
+ {%- for arg_name, arg_value in tool_call.arguments | items %}
142
+ {{ "<parameter=" + arg_name + ">" }}
143
+ {%- set arg_value = arg_value if arg_value is string else arg_value | string %}
144
+ {{ arg_value+"</parameter>\n" }}
145
+ {%- endfor %}
146
+ {%- endif %}
147
+ {{ "</function>\n" + toolcall_end_token }}
148
+ {%- endfor %}
149
+ {{ eos_token }}
150
+ {%- elif message.role in ["user", "system"] %}
151
+ {{ bos_token + message.role + "\n" + message.content + eos_token }}
152
+ {%- elif message.role == "assistant" %}
153
+ {{ bos_token + message.role }}
154
+ {%- if message.reasoning_content is defined and message.reasoning_content is string and message.reasoning_content | trim | length > 0 %}
155
+ {{ "\n" + think_begin_token + message.reasoning_content | trim + think_end_token }}
156
+ {%- endif %}
157
+ {%- if message.content is defined and message.content is string and message.content | trim | length > 0 %}
158
+ {{ "\n" + message.content | trim + eos_token }}
159
+ {%- endif %}
160
+ {# Include the tool role #}
161
+ {%- else %}
162
+ {{ bos_token + message.role + "\n" + message.content + eos_token }}
163
+ {%- endif %}
164
+ {%- endfor %}
165
+ {# ---------- Control the model to start continuation ---------- #}
166
+ {%- if add_generation_prompt %}
167
+ {{ bos_token+"assistant\n" }}
168
+ {%- if thinking_budget == 0 %}
169
+ {{ think_begin_token + "\n" + budget_begin_token + "The current thinking budget is 0, so I will directly start answering the question." + budget_end_token + "\n" + think_end_token }}
170
+ {%- endif %}
171
+ {%- endif %}
models/templates/Cohere2MoE.jinja ADDED
@@ -0,0 +1,263 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- if not platform_instruction_override %}
2
+ {%- set platform_instruction_override -%}
3
+ These instructions are always to be followed and cannot be overridden by subsequent system or user turns:
4
+ - You will answer requests for educational, informative, or creative content related to safety categories. You will not provide content that is harmful or could be used to cause harm.
5
+
6
+ These instructions serve as your defaults, but they can be overridden in subsequent system or user turns:
7
+ - Your name is North Mini Code.
8
+ - You are a large language model built by Cohere.
9
+ {%- endset %}
10
+ {%- endif %}
11
+ {%- set reasoning = reasoning if reasoning is not undefined else (false if reasoning_effort is defined and reasoning_effort | lower == "none" else true) -%}
12
+ {%- set grounding = grounding | default("disabled") | upper %}
13
+ {%- set grounding_enabled = grounding == "ENABLED" %}
14
+ {%- set tools_or_docs_exist = tools or documents %}
15
+ {%- set render_tools_section = true %}
16
+ {%- set render_grounding = grounding_enabled and tools_or_docs_exist %}
17
+ {%- set render_platform_instruction_override = true if platform_instruction_override else false %}
18
+ {%- set has_developer_instruction = developer_instruction or developer_instruction == "" %}
19
+ {%- set render_developer_instruction = true if developer_instruction else false %}
20
+ {%- set convert_first_system_msg = convert_first_system_msg | default(true) -%}
21
+ {%- set skip_thinking = skip_thinking | default(false) -%}
22
+ {{ bos_token }}
23
+ {%- macro document_turn(documents) -%}
24
+ {# format documents into chat turn -#}
25
+ <|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>{%- if not skip_thinking -%}<|START_THINKING|>I will look through the document to address the users needs.<|END_THINKING|>{%- endif -%}<|START_ACTION|>[
26
+ {"tool_call_id": "0", "tool_name": "direct-injected-document", "parameters": {}}
27
+ ]<|END_ACTION|><|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|><|START_TOOL_RESULT|>[
28
+ {
29
+ "tool_call_id": "0",
30
+ "results": {
31
+ {%- for doc in documents %}
32
+ {%- set doc_val = doc.data if doc.data else doc %}
33
+
34
+ "{{ loop.index0 }}": {{ doc_val|tojson }}{% if not loop.last %},
35
+ {%- endif %}
36
+ {%- endfor %}
37
+
38
+ },
39
+ "is_error": null
40
+ }
41
+ ]<|END_TOOL_RESULT|><|END_OF_TURN_TOKEN|>{%- endmacro %}
42
+ {%- macro tool_call_id_to_int(messages, tool_call_id) %}
43
+ {%- if regen_tool_call_ids -%}
44
+ {%- set counter = namespace(value=0) %}
45
+ {%- set tool_call_id_seen = namespace(value=false) %}
46
+ {%- for msg in messages %}
47
+ {%- if msg.tool_calls %}
48
+ {%- for tool_call in msg.tool_calls %}
49
+ {%- if tool_call.id == tool_call_id and not tool_call_id_seen.value -%}
50
+ {{ counter.value }}
51
+ {%- set tool_call_id_seen.value = true %}
52
+ {%- endif %}
53
+ {%- set counter.value = counter.value + 1 %}
54
+ {%- endfor %}
55
+ {%- endif %}
56
+ {%- endfor %}
57
+ {%- else -%}
58
+ {{ tool_call_id }}
59
+ {%- endif -%}
60
+ {%- endmacro %}
61
+ {%- macro format_tool_message(messages, tool_msg) -%}
62
+ {#- format tool message #}{
63
+ "tool_call_id": "{{ tool_call_id_to_int(messages, tool_msg.tool_call_id) }}",
64
+ "results": {
65
+ {%- if tool_msg.content is mapping or tool_msg.content is string %}
66
+
67
+ {% if tool_msg.content is string -%}
68
+ {%- set text_wrapper = {"content": tool_msg.content} -%}
69
+ {%- else -%}
70
+ {%- set text_wrapper = tool_msg.content -%}
71
+ {%- endif %}
72
+ "0": {{ text_wrapper|tojson }}
73
+ {%- else %}
74
+ {%- for content in tool_msg.content %}
75
+
76
+ "{{ loop.index0 }}": {{ print_tool_content(content) }}{% if not loop.last %},{% endif %}
77
+ {%- endfor %}
78
+ {%- endif %}
79
+
80
+ },
81
+ "is_error": null
82
+ }
83
+ {%- endmacro -%}
84
+ {%- macro print_tool_content(item) %}
85
+ {%- if item.type|lower == "text" -%}
86
+ {%- set text_wrapper = {"content": item.text} -%}
87
+ {{ text_wrapper|tojson }}
88
+ {%- elif item.type|lower == "document" and item.document and "data" in item.document -%}
89
+ {{ item.document.data|tojson }}
90
+ {%- else -%}
91
+ {{ item|tojson }}
92
+ {%- endif -%}
93
+ {%- endmacro %}
94
+ {%- macro print_msg(msg) %}
95
+ {%- if msg is string -%}
96
+ <|START_TEXT|>{{ msg }}<|END_TEXT|>
97
+ {%- elif msg.content is string -%}
98
+ <|START_TEXT|>{{ msg.content }}<|END_TEXT|>
99
+ {%- else %}
100
+ {%- set last_was_text = namespace(value=false) %}
101
+ {%- for content in msg.content %}
102
+ {%- if content.type|lower == "text" -%}
103
+ {%- if not last_was_text.value -%}
104
+ <|START_TEXT|>
105
+ {%- endif -%}
106
+ {{ content.text }}
107
+ {%- if loop.last -%}
108
+ <|END_TEXT|>
109
+ {%- endif %}
110
+ {%- set last_was_text.value = true -%}
111
+ {%- else -%}
112
+ {%- if last_was_text.value -%}
113
+ <|END_TEXT|>
114
+ {%- endif -%}
115
+ {%- set last_was_text.value = false -%}
116
+ {%- endif -%}
117
+ {%- if content.type|lower == "image" -%}
118
+ {%- if content.data -%}
119
+ {{ content.data }}
120
+ {%- else -%}
121
+ <|IMG_PATCH|>
122
+ {%- endif -%}
123
+ {%- endif -%}
124
+ {%- endfor %}
125
+ {%- endif %}
126
+ {%- endmacro %}
127
+ {%- macro print_thinking(msg) %}
128
+ {%- if msg.reasoning -%}
129
+ {{ msg.reasoning }}
130
+ {%- elif msg.reasoning_content -%}
131
+ {{ msg.reasoning_content }}
132
+ {%- elif msg.thinking -%}
133
+ {{ msg.thinking }}
134
+ {%- elif msg.content and msg.content[0].thinking -%}
135
+ {{ msg.content[0].thinking }}
136
+ {%- endif %}
137
+ {%- endmacro %}
138
+ {%- if messages and messages[0]['role']|lower == 'system' and not has_developer_instruction and convert_first_system_msg %}{%- set developer_instruction = messages[0] %}{%- set render_developer_instruction = true %}{%- set initial_instruction_message = true %}{% endif %}
139
+ {%- set json_object = true if response_format and response_format.type == "json_object" else false %}
140
+ {%- set json_schema = (response_format.json_schema or response_format.schema) if response_format %}
141
+ {%- set json_mode = json_object or json_schema %}
142
+ {%- set tool_idx = namespace(value=0) %}
143
+ {%- set tool_ids_seen = namespace(value=[]) %}
144
+ {%- set regen_tool_call_ids = regen_tool_call_ids | default(true) -%}
145
+ {%- set sent_documents = namespace(value=false) -%}
146
+
147
+ {%- if render_tools_section or render_platform_instruction_override or render_grounding or json_mode -%}
148
+ <|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|><|START_TEXT|>
149
+ {%- elif not render_developer_instruction -%}
150
+ <|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>
151
+ {%- endif %}
152
+
153
+ {%- set rendered_platform_turn_chunk = false %}
154
+
155
+ {%- if render_platform_instruction_override -%}
156
+ {{ platform_instruction_override }}
157
+ {% set rendered_platform_turn_chunk = true %}
158
+ {%- else %}
159
+ {%- endif %}
160
+
161
+ {%- if render_grounding -%}
162
+ {%- if rendered_platform_turn_chunk %}
163
+
164
+ {% endif -%}
165
+ Note that both your responses and reflections can be grounded. Grounding means you associate pieces of texts (called "spans") with those specific tool results that support them (called "sources"). And you use a pair of tags "<co>" and "</co>" to indicate when a span can be grounded onto a list of sources, listing them out in the closing tag. Sources from the same tool call are grouped together and listed as "{tool_call_id}:[{list of result indices}]", before they are joined together by ",". E.g., "<co>span</co: 0:[1,2],1:[0]>" means that "span" is supported by result 1 and 2 from "tool_call_id=0" as well as result 0 from "tool_call_id=1".
166
+ {% set rendered_platform_turn_chunk = true %}
167
+ {%- endif %}
168
+
169
+ {%- if render_tools_section %}
170
+ {%- if rendered_platform_turn_chunk %}
171
+
172
+ {% endif %}
173
+ # Available Tools
174
+ ```json
175
+ [
176
+ {% if tools_or_docs_exist %}
177
+ {%- if documents %}
178
+ {"name": "direct-injected-document", "description": "This is a special tool to directly inject user-uploaded documents into the chat as additional context. DO NOT use this tool by yourself!", "parameters": {"type": "object", "properties": {}, "required": []}, "responses": {"200": {"description": "Successfully returned a list of chunked text snippets from the directly uploaded documents.", "content": {"application/json": {"schema": {"type": "array", "items": {"type": "object", "required": ["url", "snippet"], "properties": {"url": {"type": "string", "description": "The url of the uploaded document."}, "snippet": {"type": "string", "description": "The text snippet for the returned document chunk."}}}}}}}}}
179
+ {%- if tools %},
180
+ {% else %}
181
+
182
+ {% endif %}
183
+ {%- endif %}
184
+ {%- for tool in tools %}
185
+ {"name": "{{ tool['function']['name'] }}", "description": "{{ tool['function']['description'] }}", "parameters": {{ tool['function']['parameters']|tojson }}, "responses": null}
186
+ {%- if not loop.last %},{% endif %}
187
+
188
+ {% endfor %}
189
+ {%- else %}
190
+
191
+ {% endif %}
192
+ ]
193
+ ```
194
+ {%- set rendered_platform_turn_chunk = true %}
195
+ {%- endif -%}
196
+
197
+ {%- if json_mode -%}
198
+ {%- if rendered_platform_turn_chunk %}
199
+
200
+
201
+ {% endif -%}
202
+ When generating JSON objects, do not generate block markers. Generate an object directly without prefixing with ```json. Return only the JSON and nothing else.
203
+ {%- if json_schema %}
204
+
205
+ Your output should adhere to the following json schema:
206
+ {{ json_schema }}
207
+ {%- endif -%}
208
+ {%- set rendered_platform_turn_chunk = true %}
209
+ {%- endif %}
210
+ {%- if rendered_platform_turn_chunk -%}
211
+ <|END_TEXT|><|END_OF_TURN_TOKEN|>
212
+ {%- elif not render_developer_instruction -%}
213
+ <|END_OF_TURN_TOKEN|>
214
+ {%- endif %}
215
+ {%- if render_developer_instruction -%}
216
+ <|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>{{ print_msg(developer_instruction) }}<|END_OF_TURN_TOKEN|>
217
+ {%- endif %}
218
+ {%- for message in messages %}
219
+ {%- set msg_role_downcased = message.role | lower %}
220
+ {%- if msg_role_downcased == 'system' and (not (loop.first and initial_instruction_message)) -%}
221
+ <|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>{{ print_msg(message) }}<|END_OF_TURN_TOKEN|>
222
+ {%- elif msg_role_downcased == 'user' -%}
223
+ <|START_OF_TURN_TOKEN|><|USER_TOKEN|>{{ print_msg(message) }}<|END_OF_TURN_TOKEN|>
224
+ {%- if documents and not sent_documents.value %}{%- set sent_documents.value = true %}{% set tool_idx.value = tool_idx.value + 1 %}{{ document_turn(documents) }}{% endif %}
225
+ {%- elif msg_role_downcased == 'assistant' or msg_role_downcased == 'chatbot' -%}
226
+ <|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>
227
+ {%- if message.tool_calls %}
228
+ {% if not skip_thinking %}
229
+ {% if message.tool_plan -%}
230
+ <|START_THINKING|>{{ message.tool_plan }}<|END_THINKING|>
231
+ {%- elif message.reasoning or message.reasoning_content or message.thinking or (message.content and message.content[0].type == "thinking") -%}
232
+ <|START_THINKING|>{{ print_thinking(message) }}<|END_THINKING|>
233
+ {%- endif %}
234
+ {%- endif %}<|START_ACTION|>[
235
+ {%- for tc in message.tool_calls %}
236
+
237
+ {"tool_call_id": "{%- if regen_tool_call_ids -%}{{ tool_idx.value }}{%- else -%}{{ tc.id }}{%- endif -%}", "tool_name": "{{ tc['function']['name'] }}", "parameters": {{ tc['function']['arguments']|tojson }}}{% if not loop.last %},{% endif %}
238
+ {%- set tool_idx.value = tool_idx.value + 1 %}
239
+ {%- endfor %}
240
+
241
+ ]<|END_ACTION|><|END_OF_TURN_TOKEN|>
242
+ {%- else -%}
243
+ {% if (message.reasoning or message.reasoning_content or message.thinking or (message.content and message.content[0].type == "thinking")) and not skip_thinking -%}
244
+ <|START_THINKING|>{{ print_thinking(message) }}<|END_THINKING|>
245
+ {%- endif -%}
246
+ {{ print_msg(message) }}<|END_OF_TURN_TOKEN|>
247
+ {%- endif %}
248
+ {%- elif msg_role_downcased == 'tool' and message.tool_call_id not in tool_ids_seen.value -%}
249
+ <|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|><|START_TOOL_RESULT|>[
250
+ {{ format_tool_message(messages, message) }}
251
+ {%- for msg in messages[loop.index0 + 1:] %}
252
+
253
+ {%- if msg.role | lower == 'tool' %},
254
+ {{ format_tool_message(messages, msg) }}
255
+ {%- set tool_ids_seen.value = tool_ids_seen.value + [msg.tool_call_id] %}
256
+ {%- else %}
257
+ {%- break %}
258
+ {%- endif %}
259
+ {%- endfor %}
260
+
261
+ ]<|END_TOOL_RESULT|><|END_OF_TURN_TOKEN|>
262
+ {%- endif %}
263
+ {%- endfor %}{%- if add_generation_prompt -%}<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>{% if reasoning %}<|START_THINKING|>{% else %}<|START_THINKING|><|END_THINKING|>{% endif %}{%- endif %}
models/templates/CohereForAI-c4ai-command-r-plus-tool_use.jinja ADDED
@@ -0,0 +1,202 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+
2
+ {%- macro json_to_python_type(json_spec) %}
3
+ {%- set basic_type_map = {
4
+ "string": "str",
5
+ "number": "float",
6
+ "integer": "int",
7
+ "boolean": "bool"
8
+ } %}
9
+
10
+ {%- if basic_type_map[json_spec.type] is defined %}
11
+ {{- basic_type_map[json_spec.type] }}
12
+ {%- elif json_spec.type == "array" %}
13
+ {{- "List[" + json_to_python_type(json_spec.items) + "]"}}
14
+ {%- elif json_spec.type == "object" %}
15
+ {{- "Dict[str, " + json_to_python_type(json_spec.additionalProperties) + ']'}}
16
+ {%- elif json_spec.type is iterable %}
17
+ {{- "Union[" }}
18
+ {%- for t in json_spec.type %}
19
+ {{- json_to_python_type({"type": t}) }}
20
+ {%- if not loop.last %}
21
+ {{- "," }}
22
+ {%- endif %}
23
+ {%- endfor %}
24
+ {{- "]" }}
25
+ {%- else %}
26
+ {{- "Any" }}
27
+ {%- endif %}
28
+ {%- endmacro %}
29
+
30
+ {%- macro old_tool_parser(tools) %}
31
+ {%- for tool in tools %}
32
+ {%- if loop.index0 != 0 %}
33
+ {{- '\n\n' }}
34
+ {%- endif %}
35
+ {{- '```python\ndef ' + tool.name + '(' }}
36
+ {%- for param_name, param_fields in tool.parameter_definitions|items %}
37
+ {%- if loop.index0 != 0 %}
38
+ {{- ', '}}
39
+ {%- endif %}
40
+ {{- param_name + ': ' }}
41
+ {%- if not param_fields.required %}
42
+ {{- 'Optional[' + param_fields.type + '] = None'}}
43
+ {%- else %}
44
+ {{- param_fields.type }}
45
+ {%- endif %}
46
+ {%- endfor %}
47
+ {{- ') -> List[Dict]:\n """'}}
48
+ {{- tool.description }}
49
+ {%- if tool.parameter_definitions|length != 0 %}
50
+ {{- '\n\n Args:\n '}}
51
+ {%- for param_name, param_fields in tool.parameter_definitions|items %}
52
+ {%- if loop.index0 != 0 %}
53
+ {{- '\n ' }}
54
+ {%- endif %}
55
+ {{- param_name + ' ('}}
56
+ {%- if not param_fields.required %}
57
+ {{- 'Optional[' + param_fields.type + ']'}}
58
+ {%- else %}
59
+ {{- param_fields.type }}
60
+ {%- endif %}
61
+ {{- '): ' + param_fields.description }}
62
+ {%- endfor %}
63
+ {%- endif %}
64
+ {{- '\n """\n pass\n```' }}
65
+ {%- endfor %}
66
+ {%- endmacro %}
67
+
68
+ {%- macro new_tool_parser(tools) %}
69
+ {%- for tool in tools %}
70
+ {%- if loop.index0 != 0 %}
71
+ {{- '\n\n'}}
72
+ {%- endif %}
73
+ {%- if tool.function is defined %}
74
+ {%- set tool = tool.function %}
75
+ {%- endif %}
76
+ {{-'```python
77
+ def ' + tool.name + '('}}
78
+ {%- for param_name, param_fields in tool.parameters.properties|items %}
79
+ {%- if loop.index0 != 0 %}
80
+ {{- ', '}}
81
+ {%- endif %}
82
+ {{-param_name + ": "}}
83
+ {%- if not param_name in tool.parameters.required %}
84
+ {{-'Optional[' + json_to_python_type(param_fields) + '] = None'}}
85
+ {%- else %}
86
+ {{- json_to_python_type(param_fields) }}
87
+ {%- endif %}
88
+ {%- endfor %}
89
+ {{- ') -> List[Dict]:
90
+ """'}}
91
+ {{- tool.description }}
92
+ {%- if tool.parameters.properties|length != 0 %}
93
+ {{- '\n\n Args:\n '}}
94
+ {%- for param_name, param_fields in tool.parameters.properties|items %}
95
+ {%- if loop.index0 != 0 %}
96
+ {{- '\n ' }}
97
+ {%- endif %}
98
+ {{- param_name + ' ('}}
99
+ {%- if not param_name in tool.parameters.required %}
100
+ {{-'Optional[' + json_to_python_type(param_fields) + ']'}}
101
+ {%- else %}
102
+ {{- json_to_python_type(param_fields) }}
103
+ {%- endif %}
104
+ {{- '): ' + param_fields.description }}
105
+ {%- endfor %}
106
+ {%- endif %}
107
+ {{- '\n """\n pass\n```' }}
108
+ {%- endfor %}
109
+ {%- endmacro %}
110
+
111
+ {{- bos_token }}
112
+ {%- if messages[0]['role'] == 'system' %}
113
+ {%- set loop_messages = messages[1:] %}
114
+ {%- set system_message = messages[0]['content'] %}
115
+ {%- else %}
116
+ {%- set loop_messages = messages %}
117
+ {%- set system_message = '## Task and Context\nYou help people answer their questions and other requests interactively. You will be asked a very wide array of requests on all kinds of topics. You will be equipped with a wide range of search engines or similar tools to help you, which you use to research your answer. You should focus on serving the user\'s needs as best you can, which will be wide-ranging.\n\n## Style Guide\nUnless the user asks for a different style of answer, you should answer in full sentences, using proper grammar and spelling.' %}
118
+ {%- endif %}
119
+ {{- '<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>' }}
120
+ {{- '# Safety Preamble' }}
121
+ {{- '
122
+ The instructions in this section override those in the task description and style guide sections. Don\'t answer questions that are harmful or immoral.' }}
123
+ {{- '
124
+
125
+ # System Preamble' }}
126
+ {{- '
127
+ ## Basic Rules' }}
128
+ {{- '
129
+ You are a powerful conversational AI trained by Cohere to help people. You are augmented by a number of tools, and your job is to use and consume the output of these tools to best help the user. You will see a conversation history between yourself and a user, ending with an utterance from the user. You will then see a specific instruction instructing you what kind of response to generate. When you answer the user\'s requests, you cite your sources in your answers, according to those instructions.' }}
130
+ {{- '
131
+
132
+ # User Preamble' }}
133
+ {{- '
134
+ ' + system_message }}
135
+ {{-'
136
+
137
+ ## Available Tools
138
+ Here is a list of tools that you have available to you:
139
+
140
+ '}}
141
+ {%- set ns = namespace(new_tools=true) %}
142
+ {%- for tool in tools %}
143
+ {%- if tool.parameter_definitions is defined %}
144
+ {%- set ns.new_tools = false %}
145
+ {%- endif %}
146
+ {%- endfor %}
147
+ {%- if ns.new_tools %}
148
+ {{- new_tool_parser(tools) }}
149
+ {%- else %}
150
+ {{- old_tool_parser(tools) }}
151
+ {%- endif %}
152
+ {{- '<|END_OF_TURN_TOKEN|>'}}
153
+ {%- for message in loop_messages %}
154
+ {%- set content = message['content'] %}
155
+ {%- if message.role == 'user' %}
156
+ {{- '<|START_OF_TURN_TOKEN|><|USER_TOKEN|>' + content|trim + '<|END_OF_TURN_TOKEN|>' }}
157
+ {%- elif message.role == 'system' %}
158
+ {{- '<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>' + content|trim + '<|END_OF_TURN_TOKEN|>' }}
159
+ {%- elif message.role == 'assistant' and message.tool_calls is defined %}
160
+ {{- '<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>' }}
161
+ {%- if message.content is defined %}
162
+ {{- message.content|trim }}
163
+ {%- endif %}
164
+ {{- '\nAction:\n```json\n[\n' }}
165
+ {%- for tool_call in message.tool_calls %}
166
+ {%- if tool_call.function is defined %}
167
+ {%- set tool_call = tool_call.function %}
168
+ {%- endif %}
169
+ {{- '{\n'|indent(4, first=true) }}
170
+ {{- '"tool_name": "'|indent(8, first=true) + tool_call.name + '",\n' }}
171
+ {{- '"parameters": '|indent(8, first=true) }}
172
+ {%- if tool_call.arguments is defined and tool_call.arguments|length > 0 %}
173
+ {{- tool_call.arguments|tojson(indent=4)|indent(8) }}
174
+ {{- '\n' }}
175
+ {%- else %}
176
+ {{- '{}\n' }}
177
+ {%- endif %}
178
+ {{- '}'|indent(4, first=true) }}
179
+ {%- if not loop.last %}
180
+ {{- ',\n' }}
181
+ {%- endif %}
182
+ {%- endfor %}
183
+ {{- "\n]```\n" }}
184
+ {%- elif message.role == 'assistant' %}
185
+ {{- '<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>' + content|trim + '<|END_OF_TURN_TOKEN|>' }}
186
+ {%- elif message.role == 'tool' %}
187
+ {{- '<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|><results>\n' }}
188
+ {{- message.content|trim }}
189
+ {{- '</results><|END_OF_TURN_TOKEN|>' }}
190
+ {%- endif %}
191
+ {%- endfor %}
192
+ {{-'<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>Write \'Action:\' followed by a json-formatted list of actions that you want to perform in order to produce a good response to the user\'s last input. You can use any of the supplied tools any number of times, but you should aim to execute the minimum number of necessary actions for the input. You should use the `directly-answer` tool if calling the other tools is unnecessary. The list of actions you want to call should be formatted as a list of json objects, for example:
193
+ ```json
194
+ [
195
+ {
196
+ "tool_name": title of the tool in the specification,
197
+ "parameters": a dict of parameters to input into the tool as they are defined in the specs, or {} if it takes no parameters
198
+ }
199
+ ]```<|END_OF_TURN_TOKEN|>'}}
200
+ {%- if add_generation_prompt %}
201
+ {{- '<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>' }}
202
+ {%- endif %}
models/templates/CohereForAI-c4ai-command-r7b-12-2024-tool_use.jinja ADDED
@@ -0,0 +1,156 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {{ bos_token }}{%- macro document_turn(documents) -%}
2
+ {# format documents into chat turn #}
3
+ <|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|><|START_THINKING|>I will look through the document to address the users needs.<|END_THINKING|><|START_ACTION|>[
4
+ {"tool_call_id": "0", "tool_name": "direct-injected-document", "parameters": {}}
5
+ ]<|END_ACTION|><|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|><|START_TOOL_RESULT|>[
6
+ {
7
+ "tool_call_id": "0",
8
+ "results": {
9
+ {% for doc in documents %}
10
+ "{{ loop.index0 }}": {{doc|tojson}}{% if not loop.last %},
11
+ {% endif %}
12
+ {% endfor %}
13
+
14
+ },
15
+ "is_error": null
16
+ }
17
+ ]<|END_TOOL_RESULT|><|END_OF_TURN_TOKEN|>{%- endmacro %}
18
+ {%- macro tool_call_id_to_int(messages, tool_call_id) %}
19
+ {%- set counter = namespace(value=0) %}
20
+ {%- set tool_call_id_seen = namespace(value=false) %}
21
+ {%- for msg in messages %}
22
+ {%- if msg.tool_calls %}
23
+ {%- for tool_call in msg.tool_calls %}
24
+ {%- if tool_call.id == tool_call_id and not tool_call_id_seen.value -%}
25
+ {{ counter.value }}
26
+ {%- set tool_call_id_seen.value = true %}
27
+ {%- endif %}
28
+ {%- set counter.value = counter.value + 1 %}
29
+ {%- endfor %}
30
+ {%- endif %}
31
+ {%- endfor %}
32
+ {%- endmacro %}
33
+ {%- macro format_tool_message(messages, tool_msg) -%}
34
+ {# format tool message #}
35
+ {
36
+ "tool_call_id": "{{ tool_call_id_to_int(messages, tool_msg.tool_call_id) }}",
37
+ "results": {
38
+ "0": {{ tool_msg.content|tojson }}
39
+ },
40
+ "is_error": null
41
+ }
42
+ {%- endmacro -%}
43
+ {%- if messages and messages[0]['role']|lower == 'system' %}{%- set developer_preamble = messages[0]['content'] %}{% endif %}
44
+ {%- set tool_idx = namespace(value=0) %}
45
+ {%- set tool_ids_seen = namespace(value=[]) %}
46
+ {%- set sent_documents = namespace(value=false) %}
47
+ <|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|># System Preamble
48
+ You are in contextual safety mode. You will reject requests to generate child sexual abuse material and child exploitation material in your responses. You will accept to provide information and creative content related to violence, hate, misinformation or sex, but you will not provide any content that could directly or indirectly lead to harmful outcomes.
49
+
50
+ Your information cutoff date is June 2024.
51
+
52
+ You have been trained on data in English, French, Spanish, Italian, German, Portuguese, Japanese, Korean, Modern Standard Arabic, Mandarin, Russian, Indonesian, Turkish, Dutch, Polish, Persian, Vietnamese, Czech, Hindi, Ukrainian, Romanian, Greek and Hebrew but have the ability to speak many more languages.
53
+ {% if tools or documents %}
54
+
55
+ You have been trained to have advanced reasoning and tool-use capabilities and you should make best use of these skills to serve user's requests.
56
+
57
+ ## Tool Use
58
+ Think about how you can make best use of the provided tools to help with the task and come up with a high level plan that you will execute first.
59
+
60
+ 0. Start by writing <|START_THINKING|> followed by a detailed step by step plan of how you will solve the problem. For each step explain your thinking fully and give details of required tool calls (if needed). Unless specified otherwise, you write your plan in natural language. When you finish, close it out with <|END_THINKING|>.
61
+ You can optionally choose to skip this step when the user request is so straightforward to address that only a trivial plan would be needed.
62
+ NOTE: You MUST skip this step when you are directly responding to the user's request without using any tools.
63
+
64
+ Then carry out your plan by repeatedly executing the following steps.
65
+ 1. Action: write <|START_ACTION|> followed by a list of JSON-formatted tool calls, with each one containing "tool_name" and "parameters" fields.
66
+ When there are multiple tool calls which are completely independent of each other (i.e. they can be executed in parallel), you should list them out all together in one step. When you finish, close it out with <|END_ACTION|>.
67
+ 2. Observation: you will then receive results of those tool calls in JSON format in the very next turn, wrapped around by <|START_TOOL_RESULT|> and <|END_TOOL_RESULT|>. Carefully observe those results and think about what to do next. Note that these results will be provided to you in a separate turn. NEVER hallucinate results.
68
+ Every tool call produces a list of results (when a tool call produces no result or a single result, it'll still get wrapped inside a list). Each result is clearly linked to its originating tool call via its "tool_call_id".
69
+ 3. Reflection: start the next turn by writing <|START_THINKING|> followed by what you've figured out so far, any changes you need to make to your plan, and what you will do next. When you finish, close it out with <|END_THINKING|>.
70
+ You can optionally choose to skip this step when everything is going according to plan and no special pieces of information or reasoning chains need to be recorded.
71
+ NOTE: You MUST skip this step when you are done with tool-use actions and are ready to respond to the user.
72
+
73
+ You can repeat the above 3 steps multiple times (could be 0 times too if no suitable tool calls are available or needed), until you decide it's time to finally respond to the user.
74
+
75
+ 4. Response: then break out of the loop and write <|START_RESPONSE|> followed by a piece of text which serves as a response to the user's last request. Use all previous tool calls and results to help you when formulating your response. When you finish, close it out with <|END_RESPONSE|>.
76
+ {% if enable_citations %}
77
+
78
+ ## Grounding
79
+ Importantly, note that "Reflection" and "Response" above can be grounded.
80
+ Grounding means you associate pieces of texts (called "spans") with those specific tool results that support them (called "sources"). And you use a pair of tags "<co>" and "</co>" to indicate when a span can be grounded onto a list of sources, listing them out in the closing tag. Sources from the same tool call are grouped together and listed as "{tool_call_id}:[{list of result indices}]", before they are joined together by ",". E.g., "<co>span</co: 0:[1,2],1:[0]>" means that "span" is supported by result 1 and 2 from "tool_call_id=0" as well as result 0 from "tool_call_id=1".
81
+ {% endif %}
82
+
83
+ ## Available Tools
84
+ Here is the list of tools that you have available to you.
85
+ You can ONLY use the tools listed here. When a tool is not listed below, it is NOT available and you should NEVER attempt to use it.
86
+ Each tool is represented as a JSON object with fields like "name", "description", "parameters" (per JSON Schema), and optionally, "responses" (per JSON Schema).
87
+
88
+ ```json
89
+ [
90
+ {% if documents %}
91
+ {"name": "direct-injected-document", "description": "This is a special tool to directly inject user-uploaded documents into the chat as additional context. DO NOT use this tool by yourself!", "parameters": {"type": "object", "properties": {}, "required": []}, "responses": {"200": {"description": "Successfully returned a list of chunked text snippets from the directly uploaded documents.", "content": {"application/json": {"schema": {"type": "array", "items": {"type": "object", "required": ["url", "snippet"], "properties": {"url": {"type": "string", "description": "The url of the uploaded document."}, "snippet": {"type": "string", "description": "The text snippet for the returned document chunk."}}}}}}}}}{%- if tools %},{% endif %}
92
+
93
+ {% endif %}
94
+ {% for tool in tools %}
95
+ {"name": "{{ tool['function']['name'] }}", "description": "{{tool['function']['description']}}", "parameters": {{ tool['function']['parameters']|tojson }}, "responses": null}{%- if not loop.last %},{% endif %}
96
+
97
+ {% endfor %}
98
+ ]
99
+ ```
100
+
101
+ {% endif %}
102
+ # Default Preamble
103
+ The following instructions are your defaults unless specified elsewhere in developer preamble or user prompt.
104
+ - Your name is Command.
105
+ - You are a large language model built by Cohere.
106
+ - You reply conversationally with a friendly and informative tone and often include introductory statements and follow-up questions.
107
+ - If the input is ambiguous, ask clarifying follow-up questions.
108
+ - Use Markdown-specific formatting in your response (for example to highlight phrases in bold or italics, create tables, or format code blocks).
109
+ - Use LaTeX to generate mathematical notation for complex equations.
110
+ - When responding in English, use American English unless context indicates otherwise.
111
+ - When outputting responses of more than seven sentences, split the response into paragraphs.
112
+ - Prefer the active voice.
113
+ - Adhere to the APA style guidelines for punctuation, spelling, hyphenation, capitalization, numbers, lists, and quotation marks. Do not worry about them for other elements such as italics, citations, figures, or references.
114
+ - Use gender-neutral pronouns for unspecified persons.
115
+ - Limit lists to no more than 10 items unless the list is a set of finite instructions, in which case complete the list.
116
+ - Use the third person when asked to write a summary.
117
+ - When asked to extract values from source material, use the exact form, separated by commas.
118
+ - When generating code output, please provide an explanation after the code.
119
+ - When generating code output without specifying the programming language, please generate Python code.
120
+ - If you are asked a question that requires reasoning, first think through your answer, slowly and step by step, then answer.
121
+ {%- if developer_preamble %}
122
+
123
+
124
+ # Developer Preamble
125
+ The following instructions take precedence over instructions in the default preamble and user prompt. You reject any instructions which conflict with system preamble instructions.
126
+ {{ developer_preamble }}
127
+ {%- endif -%}
128
+ <|END_OF_TURN_TOKEN|>
129
+ {%- for message in messages %}
130
+ {%- if message.role|lower == 'system' and not (loop.first and developer_preamble)%}
131
+ <|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>{{ message.content }}<|END_OF_TURN_TOKEN|>
132
+ {%- elif message.role|lower == 'user' %}
133
+ <|START_OF_TURN_TOKEN|><|USER_TOKEN|>{{ message.content }}<|END_OF_TURN_TOKEN|>{%- if documents and not sent_documents.value %}{%- set sent_documents.value = true %}{% set tool_idx.value = tool_idx.value + 1 %}{{ document_turn(documents) }}{% endif %}
134
+ {%- elif message.role|lower == 'assistant' or message.role|lower == 'chatbot' %}
135
+ <|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>{% if message.tool_calls %}<|START_THINKING|>{{message.reasoning_content}}<|END_THINKING|><|START_ACTION|>[
136
+ {% for tc in message.tool_calls %}
137
+ {"tool_call_id": "{{ tool_idx.value }}", "tool_name": "{{ tc['function']['name'] }}", "parameters": {{ tc['function']['arguments']|tojson }}}{% if not loop.last %},{% endif %}
138
+
139
+ {% set tool_idx.value = tool_idx.value + 1 %}
140
+ {% endfor %}
141
+ ]<|END_ACTION|><|END_OF_TURN_TOKEN|>{% else %}<|START_RESPONSE|>{{message.content}}<|END_RESPONSE|><|END_OF_TURN_TOKEN|>{% endif %}
142
+ {% elif message.role|lower == 'tool' and message.tool_call_id not in tool_ids_seen.value %}
143
+ <|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|><|START_TOOL_RESULT|>[
144
+ {{ format_tool_message(messages, message) }}
145
+ {%- for msg in messages[loop.index0 + 1:] %}
146
+ {%- if msg.role|lower == 'tool' %},
147
+ {{ format_tool_message(messages, msg) }}
148
+ {%- set tool_ids_seen.value = tool_ids_seen.value + [msg.tool_call_id] %}
149
+ {%- else %}
150
+ {%- break %}
151
+ {%- endif %}
152
+ {%- endfor %}
153
+
154
+ ]<|END_TOOL_RESULT|><|END_OF_TURN_TOKEN|>
155
+ {%- endif %}
156
+ {%- endfor %}<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>{%- if not enable_thinking -%}<|START_THINKING|><|END_THINKING|>{%- endif %}
models/templates/GLM-4.6.jinja ADDED
@@ -0,0 +1,106 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [gMASK]<sop>
2
+ {%- if tools -%}
3
+ <|system|>
4
+ # Tools
5
+
6
+ You may call one or more functions to assist with the user query.
7
+
8
+ You are provided with function signatures within <tools></tools> XML tags:
9
+ <tools>
10
+ {% for tool in tools %}
11
+ {{ tool | tojson(ensure_ascii=False) }}
12
+ {% endfor %}
13
+ </tools>
14
+
15
+ For each function call, output the function name and arguments within the following XML format:
16
+ <tool_call>{function-name}
17
+ <arg_key>{arg-key-1}</arg_key>
18
+ <arg_value>{arg-value-1}</arg_value>
19
+ <arg_key>{arg-key-2}</arg_key>
20
+ <arg_value>{arg-value-2}</arg_value>
21
+ ...
22
+ </tool_call>{%- endif -%}
23
+ {%- macro visible_text(content) -%}
24
+ {%- if content is string -%}
25
+ {{- content }}
26
+ {%- elif content is iterable and content is not mapping -%}
27
+ {%- for item in content -%}
28
+ {%- if item is mapping and item.type == 'text' -%}
29
+ {{- item.text }}
30
+ {%- elif item is string -%}
31
+ {{- item }}
32
+ {%- endif -%}
33
+ {%- endfor -%}
34
+ {%- else -%}
35
+ {{- content }}
36
+ {%- endif -%}
37
+ {%- endmacro -%}
38
+ {%- set ns = namespace(last_user_index=-1) %}
39
+ {%- for m in messages %}
40
+ {%- if m.role == 'user' %}
41
+ {% set ns.last_user_index = loop.index0 -%}
42
+ {%- endif %}
43
+ {%- endfor %}
44
+ {% for m in messages %}
45
+ {%- if m.role == 'user' -%}<|user|>
46
+ {{ visible_text(m.content) }}
47
+ {{- '/nothink' if (enable_thinking is defined and not enable_thinking and not visible_text(m.content).endswith("/nothink")) else '' -}}
48
+ {%- elif m.role == 'assistant' -%}
49
+ <|assistant|>
50
+ {%- set reasoning_content = '' %}
51
+ {%- set content = visible_text(m.content) %}
52
+ {%- if m.reasoning_content is string %}
53
+ {%- set reasoning_content = m.reasoning_content %}
54
+ {%- else %}
55
+ {%- if '</think>' in content %}
56
+ {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
57
+ {%- set content = content.split('</think>')[-1].lstrip('\n') %}
58
+ {%- endif %}
59
+ {%- endif %}
60
+ {%- if loop.index0 > ns.last_user_index and reasoning_content -%}
61
+ {{ '\n<think>' + reasoning_content.strip() + '</think>'}}
62
+ {%- else -%}
63
+ {{ '\n<think></think>' }}
64
+ {%- endif -%}
65
+ {%- if content.strip() -%}
66
+ {{ '\n' + content.strip() }}
67
+ {%- endif -%}
68
+ {% if m.tool_calls %}
69
+ {% for tc in m.tool_calls %}
70
+ {%- if tc.function %}
71
+ {%- set tc = tc.function %}
72
+ {%- endif %}
73
+ {{ '\n<tool_call>' + tc.name }}
74
+ {% set _args = tc.arguments or {} %}
75
+ {% if _args is not mapping %}
76
+ {{ raise_exception("Invalid tool call arguments passed: " + _args | string) }}
77
+ {% endif %}
78
+ {% for k, v in _args.items() %}
79
+ <arg_key>{{ k }}</arg_key>
80
+ <arg_value>{{ v | tojson(ensure_ascii=False) if v is not string else v }}</arg_value>
81
+ {% endfor %}
82
+ </tool_call>{% endfor %}
83
+ {% endif %}
84
+ {%- elif m.role == 'tool' -%}
85
+ {%- if m.content is string -%}
86
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
87
+ {{- '<|observation|>' }}
88
+ {%- endif %}
89
+ {{- '\n<tool_response>\n' }}
90
+ {{- m.content }}
91
+ {{- '\n</tool_response>' }}
92
+ {%- else -%}
93
+ <|observation|>{% for tr in m.content %}
94
+
95
+ <tool_response>
96
+ {{ tr.output if tr.output is defined else tr }}
97
+ </tool_response>{% endfor -%}
98
+ {% endif -%}
99
+ {%- elif m.role == 'system' -%}
100
+ <|system|>
101
+ {{ visible_text(m.content) }}
102
+ {%- endif -%}
103
+ {%- endfor -%}
104
+ {%- if add_generation_prompt -%}
105
+ <|assistant|>{{- '\n<think></think>' if (enable_thinking is defined and not enable_thinking) else '' -}}
106
+ {%- endif -%}
models/templates/GLM-4.7-Flash.jinja ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [gMASK]<sop>
2
+ {%- if tools -%}
3
+ <|system|>
4
+ # Tools
5
+
6
+ You may call one or more functions to assist with the user query.
7
+
8
+ You are provided with function signatures within <tools></tools> XML tags:
9
+ <tools>
10
+ {% for tool in tools %}
11
+ {{ tool | tojson(ensure_ascii=False) }}
12
+ {% endfor %}
13
+ </tools>
14
+
15
+ For each function call, output the function name and arguments within the following XML format:
16
+ <tool_call>{function-name}<arg_key>{arg-key-1}</arg_key><arg_value>{arg-value-1}</arg_value><arg_key>{arg-key-2}</arg_key><arg_value>{arg-value-2}</arg_value>...</tool_call>{%- endif -%}
17
+ {%- macro visible_text(content) -%}
18
+ {%- if content is string -%}
19
+ {{- content }}
20
+ {%- elif content is iterable and content is not mapping -%}
21
+ {%- for item in content -%}
22
+ {%- if item is mapping and item.type == 'text' -%}
23
+ {{- item.text }}
24
+ {%- elif item is string -%}
25
+ {{- item }}
26
+ {%- endif -%}
27
+ {%- endfor -%}
28
+ {%- else -%}
29
+ {{- content }}
30
+ {%- endif -%}
31
+ {%- endmacro -%}
32
+ {%- set ns = namespace(last_user_index=-1) %}
33
+ {%- for m in messages %}
34
+ {%- if m.role == 'user' %}
35
+ {% set ns.last_user_index = loop.index0 -%}
36
+ {%- endif %}
37
+ {%- endfor %}
38
+ {% for m in messages %}
39
+ {%- if m.role == 'user' -%}<|user|>{{ visible_text(m.content) }}
40
+ {%- elif m.role == 'assistant' -%}
41
+ <|assistant|>
42
+ {%- set reasoning_content = '' %}
43
+ {%- set content = visible_text(m.content) %}
44
+ {%- if m.reasoning_content is string %}
45
+ {%- set reasoning_content = m.reasoning_content %}
46
+ {%- else %}
47
+ {%- if '</think>' in content %}
48
+ {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
49
+ {%- set content = content.split('</think>')[-1].lstrip('\n') %}
50
+ {%- endif %}
51
+ {%- endif %}
52
+ {%- if ((clear_thinking is defined and not clear_thinking) or loop.index0 > ns.last_user_index) and reasoning_content -%}
53
+ {{ '<think>' + reasoning_content.strip() + '</think>'}}
54
+ {%- else -%}
55
+ {{ '</think>' }}
56
+ {%- endif -%}
57
+ {%- if content.strip() -%}
58
+ {{ content.strip() }}
59
+ {%- endif -%}
60
+ {% if m.tool_calls %}
61
+ {% for tc in m.tool_calls %}
62
+ {%- if tc.function %}
63
+ {%- set tc = tc.function %}
64
+ {%- endif %}
65
+ {{- '<tool_call>' + tc.name -}}
66
+ {% set _args = tc.arguments %}{% for k, v in _args.items() %}<arg_key>{{ k }}</arg_key><arg_value>{{ v | tojson(ensure_ascii=False) if v is not string else v }}</arg_value>{% endfor %}</tool_call>{% endfor %}
67
+ {% endif %}
68
+ {%- elif m.role == 'tool' -%}
69
+ {%- if m.content is string -%}
70
+ {%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
71
+ {{- '<|observation|>' }}
72
+ {%- endif %}
73
+ {{- '<tool_response>' }}
74
+ {{- m.content }}
75
+ {{- '</tool_response>' }}
76
+ {%- else -%}
77
+ <|observation|>{% for tr in m.content %}
78
+ <tool_response>{{ tr.output if tr.output is defined else tr }}</tool_response>{% endfor -%}
79
+ {% endif -%}
80
+ {%- elif m.role == 'system' -%}
81
+ <|system|>{{ visible_text(m.content) }}
82
+ {%- endif -%}
83
+ {%- endfor -%}
84
+ {%- if add_generation_prompt -%}
85
+ <|assistant|>{{- '</think>' if (enable_thinking is defined and not enable_thinking) else '<think>' -}}
86
+ {%- endif -%}
models/templates/GigaChat3-10B-A1.8B.jinja ADDED
@@ -0,0 +1,355 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {#--------TOOL RENDERING FUNCTIONS---------#}
2
+
3
+ {#---------------------------------------------------------------
4
+ Converts JSON Schema (dict) to a TypeScript type definition
5
+ ----------------------------------------------------------------#}
6
+ {%- macro json_schema_to_typescript(schema, indent="") -%}
7
+ {%- set ADDITIONAL_JSON_KEYS = ['format', 'maxItems', 'maximum', 'minItems', 'minimum', 'pattern'] -%}
8
+ {%- set ty = schema.get("type") -%}
9
+
10
+ {# ---------------- OBJECT ---------------- #}
11
+ {%- if ty == "object" -%}
12
+ {{- "{\n" -}}
13
+
14
+ {# Start building property list #}
15
+ {%- set props = schema.get("properties", {}) -%}
16
+ {%- set required = schema.get("required", []) -%}
17
+ {%- set has_additional_props = schema.get("additionalProperties") is defined -%}
18
+ {%- set additional_props_type = none -%}
19
+ {%- if has_additional_props -%}
20
+ {%- if schema.additionalProperties == true -%}
21
+ {%- set additional_props_type = {'type': 'any'} -%}
22
+ {%- elif schema.additionalProperties is mapping -%}
23
+ {%- set additional_props_type = schema.additionalProperties -%}
24
+ {%- endif -%}
25
+ {%- endif -%}
26
+
27
+ {%- for key, val in props.items() -%}
28
+ {# ---------- Description Comments ---------- #}
29
+ {%- if "description" in val -%}
30
+ {%- for line in val['description'].split('\n') -%}
31
+ {%- if line.strip() -%}
32
+ {{- indent + '// ' + line + '\n' -}}
33
+ {%- endif -%}
34
+ {%- endfor -%}
35
+ {%- endif -%}
36
+
37
+ {# ---------- Additional JSON Keys ---------- #}
38
+ {%- for add_key, add_val in val.items() -%}
39
+ {%- if add_key in ADDITIONAL_JSON_KEYS -%}
40
+ {%- if add_val is string -%}
41
+ {{- indent + '// ' + add_key + ': "' + add_val + '"' + '\n' -}}
42
+ {%- else -%}
43
+ {{- indent + '// ' + add_key + ': ' ~ add_val ~ '\n' -}}
44
+ {%- endif -%}
45
+ {%- endif -%}
46
+ {%- endfor -%}
47
+
48
+ {# ---------- Property Definition ---------- #}
49
+ {%- set type_str = json_schema_to_typescript(
50
+ val,
51
+ indent + " "
52
+ ) -%}
53
+
54
+ {{- indent + key + ('' if key in required else '?') + ': ' + type_str + ',' -}}
55
+
56
+ {%- if "default" in val or "defalut_value" in val -%}
57
+ {%- set default = val.get("default", val.get("defalut_value")) -%}
58
+ {%- if default is string -%}
59
+ {{- ' // default: "' + default + '"' -}}
60
+ {%- else -%}
61
+ {{- ' // default: ' ~ default -}}
62
+ {%- endif -%}
63
+ {%- endif -%}
64
+
65
+ {{- "\n" -}}
66
+ {%- endfor -%}
67
+
68
+ {# Handle additionalProperties as index signature #}
69
+ {%- if has_additional_props and additional_props_type is not none -%}
70
+ {%- set additional_type_str = json_schema_to_typescript(
71
+ additional_props_type,
72
+ indent + " "
73
+ ) -%}
74
+ {{- indent + '[key: string]: ' + additional_type_str + '\n' -}}
75
+ {%- endif -%}
76
+
77
+ {{- indent[: (indent|length - " "|length) ] + '}' -}}
78
+
79
+ {# ---------------- STRING ---------------- #}
80
+ {%- elif ty == "string" -%}
81
+ {%- if schema.get("enum") -%}
82
+ {%- set ns = namespace(enum = []) -%}
83
+ {%- for en in schema['enum'] -%}
84
+ {%- set ns.enum = ns.enum + ['"' ~ en ~ '"'] -%}
85
+ {%- endfor -%}
86
+ {{- ns.enum | join(' | ') -}}
87
+ {%- elif schema.get("format", "none") in ['date-time', 'date'] -%}
88
+ {{- 'Date' -}}
89
+ {%- else -%}
90
+ {{- 'string' -}}
91
+ {%- endif -%}
92
+
93
+ {# ---------------- NUMBER / INTEGER ---------------- #}
94
+ {%- elif ty in ["number", "integer"] -%}
95
+ {%- if schema.get("enum") -%}
96
+ {{- schema.enum | join(' | ') -}}
97
+ {%- else -%}
98
+ {{- 'number' -}}
99
+ {%- endif -%}
100
+
101
+ {# ---------------- BOOLEAN ---------------- #}
102
+ {%- elif ty == "boolean" -%}
103
+ {{- 'boolean' -}}
104
+
105
+ {# ---------------- ARRAY ---------------- #}
106
+ {%- elif ty == "array" -%}
107
+ {%- if "items" in schema -%}
108
+ {{- json_schema_to_typescript(schema['items'], indent) + '[]' -}}
109
+ {%- else -%}
110
+ {{- 'Array<any>' -}}
111
+ {%- endif -%}
112
+
113
+ {# ---------------- FALLBACK ---------------- #}
114
+ {%- else -%}
115
+ {{- 'any' -}}
116
+ {%- endif -%}
117
+ {%- endmacro -%}
118
+
119
+ {#---------------------------------------------------------------
120
+ Renders a namespace and its tool definitions in TypeScript style
121
+ ----------------------------------------------------------------#}
122
+
123
+ {%- macro render_tool_namespace(namespace_name, tools) -%}
124
+ {%- set ns = namespace(sections = ['namespace ' ~ namespace_name ~ ' {']) -%}
125
+
126
+ {%- for tool in tools -%}
127
+ {%- if tool.function -%}
128
+ {%- set tool = tool.function -%}
129
+ {%- endif -%}
130
+
131
+ {%- set ns_tool = namespace(content_lines=[]) -%}
132
+
133
+ {# ---------- TOOL DESCRIPTION ---------- #}
134
+ {%- if tool.get('description') -%}
135
+ {%- for line in tool['description'].split('\n') -%}
136
+ {%- if line.strip() -%}
137
+ {%- set ns_tool.content_lines = ns_tool.content_lines + ['// ' ~ line] -%}
138
+ {%- endif -%}
139
+ {%- endfor -%}
140
+ {%- endif -%}
141
+
142
+ {# ---------- TOOL SIGNATURE ---------- #}
143
+ {%- set main_body = "" -%}
144
+ {%- set params = tool.get("parameters") -%}
145
+ {%- if params and params.get("properties") -%}
146
+ {%- set param_type = json_schema_to_typescript(params, " ") -%}
147
+ {%- set main_body = 'type ' ~ tool.name ~ ' = (_: ' ~ param_type ~ ') => ' -%}
148
+ {%- else -%}
149
+ {%- set main_body = 'type ' ~ tool.name ~ ' = () => ' -%}
150
+ {%- endif -%}
151
+
152
+ {# ---------- RETURN TYPE ---------- #}
153
+ {%- set return_params = tool.get("return_parameters") -%}
154
+ {%- if return_params and return_params.get("properties") -%}
155
+ {%- set return_type = json_schema_to_typescript(return_params, " ") -%}
156
+ {%- set main_body = main_body ~ return_type -%}
157
+ {%- else -%}
158
+ {%- set main_body = main_body ~ 'any' -%}
159
+ {%- endif -%}
160
+
161
+ {%- set main_body = main_body ~ ';\n' -%}
162
+
163
+ {%- set ns_tool.content_lines = ns_tool.content_lines + [main_body] -%}
164
+
165
+ {# ---------- ADD TOOL TO SECTIONS ---------- #}
166
+ {%- set ns.sections = ns.sections + [ns_tool.content_lines | join('\n')] -%}
167
+ {%- endfor -%}
168
+
169
+ {%- set ns.sections = ns.sections + ['} // namespace ' ~ namespace_name] -%}
170
+
171
+ {{- ns.sections | join('\n') -}}
172
+ {%- endmacro -%}
173
+
174
+
175
+ {# ----------- MESSAGE RENDERING HELPER FUNCTIONS ------------ #}
176
+
177
+ {%- macro render_role_message(message, role=None) -%}
178
+ {%- if not role -%}
179
+ {%- set role = message["role"] -%}
180
+ {%- endif -%}
181
+
182
+ {%- set message_content = message['content'] or '' -%}
183
+ {%- if message_content is not string -%}
184
+ {%- set message_content = message_content | tojson(ensure_ascii=False) -%}
185
+ {%- endif -%}
186
+
187
+ {{- role + add_tokens.role_sep + message_content + add_tokens.message_sep -}}
188
+
189
+ {%- endmacro -%}
190
+
191
+
192
+ {%- macro render_function_call(message) -%}
193
+ {%- set call = message['content'] -%}
194
+ {%- if call.function -%}
195
+ {%- set call = call.function -%}
196
+ {%- endif -%}
197
+
198
+ {%- set arguments = call['arguments'] -%}
199
+ {%- if arguments is not string -%}
200
+ {%- set arguments = arguments| tojson(ensure_ascii=False) -%}
201
+ {%- endif -%}
202
+
203
+ {{- render_role_message(
204
+ {
205
+ 'role': 'function call',
206
+ 'content': '{"name": "' ~ call['name'] ~ '", "arguments": ' ~ arguments ~ '}'
207
+ }
208
+ ) -}}
209
+ {%- endmacro -%}
210
+
211
+ {# ----- SPECIAL TOKENS ----- #}
212
+
213
+ {%- set add_tokens = namespace(
214
+ role_sep="<|role_sep|>\n",
215
+ message_sep="<|message_sep|>\n\n"
216
+ ) -%}
217
+
218
+ {# ----- DEFAULT DEVSYSTEM ----- #}
219
+
220
+ {%- set DEVSYSTEM -%}
221
+ <role_description>
222
+ Description of the roles available in the dialog.
223
+
224
+ `developer system`
225
+ A message added by Sber before the main dialog. It has the highest priority and sets global, non-overridable conditions (for example, conversation rules, the safety policy, the assistant's overall response style, etc.).
226
+
227
+ `system`
228
+ A system instruction added by developers or by the user, but with a lower priority than `developer system`. It usually describes the assistant's instructions, a specific response style, and other conditions for this particular dialog.
229
+
230
+ `user`
231
+ A message or request from the user. The assistant follows it if it does not conflict with higher-priority instructions (see <instruction_priority>).
232
+
233
+ `user memory`
234
+ A sequence of the most up-to-date long-term facts about the user at the time of their request, presented as a JSON list of strings. Facts are listed in chronological order, meaning newer facts are appended to the end of the sequence. When facts are changed or deleted, records of previous facts remain in the sequence. The assistant saves facts using a function and uses them in accordance with the <memory_guidelines> block below.
235
+
236
+ `added files`
237
+ Metadata about files available for use in the dialog, presented in JSON format. It contains the following keys: id (a unique file identifier), name (file name), type (file type).
238
+
239
+ `assistant`
240
+ The assistant's reply to the user's request. If the system instruction or the user does not set additional rules for `assistant`, this reply must comply with the instructions in the <assistant_guidelines> block below. The list of functions available to call is contained in `function descriptions`. The name of the required function and its arguments will be generated next by the `function call` role. In its replies, the assistant follows the instructions in accordance with <instruction_priority>.
241
+
242
+ `function descriptions`
243
+ Function descriptions in TypeScript format. A function is a special tool (or a set of instructions) that the assistant can call to perform specific actions, computations, or obtain data needed to solve the user's task. Each function description contains blocks with the name, description, and arguments. Sometimes the description contains separate blocks with return parameters and usage examples that illustrate the correct call and arguments.
244
+
245
+ `function call`
246
+ The function that `assistant` calls based on the dialog context, and its arguments. The function is invoked in strict accordance with the instructions in the <function_usage> block.
247
+
248
+ `function result`
249
+ The result of the last function call.
250
+ </role_description>
251
+
252
+ <available_modalities>
253
+ The assistant can work with the following modalities: text, available functions.
254
+ </available_modalities>
255
+
256
+ <instruction_priority>
257
+ If instructions from different roles conflict within the dialog context, observe the following priorities:
258
+ `developer system` > `system` > `user` > `function descriptions` > `function result` > `user memory`
259
+ </instruction_priority>
260
+
261
+ <function_usage>
262
+ Basic instructions for working with functions.
263
+
264
+ Only call those functions that are described in `function descriptions`.
265
+
266
+ Call available functions when, according to their description, such a call will help provide a more complete and/or accurate answer to the user's request. Fill in function arguments using information from the dialog context. If a function could help answer the request but a required argument is missing from the context, ask the user for the missing data before calling the function. If a necessary function is unavailable or an error occurs, briefly inform the user and, if possible, suggest an alternative.
267
+ </function_usage>
268
+
269
+ <memory_guidelines>
270
+ Rules for using facts in long-term memory:
271
+
272
+ If there is no message under the `user memory` role in the dialog, this is equivalent to the absence of long-term facts about the user in memory. In that case, information about the user is limited to the current dialog, and no new facts should be saved.
273
+ </memory_guidelines>
274
+
275
+ <assistant_guidelines>
276
+ You are a helpful assistant.
277
+
278
+ # Instructions
279
+ - Strictly follow the instruction priority.
280
+ - Maintain a logical chain of reasoning when answering the user's question.
281
+ - For complex questions (for example, STEM), try to answer in detail unless the system message or dialog context limits the response length.
282
+ - Be helpful, truthful, and avoid unsafe or prohibited content in your responses.
283
+ - Try to reply in the language in which the user asked their question.
284
+ </assistant_guidelines>
285
+
286
+ A dialog will follow below.
287
+ The dialog may include various roles described in the <role_description> block.
288
+ Each turn begins with the role name and a special token that marks the end of the role's full name, and ends with a special end-of-turn token.
289
+ Your task is to continue the dialog from the last specified role in accordance with the dialog context.
290
+ {%- endset -%}
291
+
292
+
293
+ {#- ---------------------- RENDERING STARTS HERE ---------------------- -#}
294
+
295
+
296
+ {# ----- RENDER BOS TOKEN ----- #}
297
+ {{- bos_token -}}
298
+
299
+
300
+ {# ----- RENDER DEVSYSTEM ----- #}
301
+ {{- render_role_message({"role": "developer system", "content": DEVSYSTEM}) -}}
302
+
303
+ {# ----- RENDER SYSTEM IF PRESENT ----- #}
304
+ {%- if messages and messages[0]['role'] == 'system' -%}
305
+ {{- render_role_message(messages[0]) -}}
306
+ {%- set messages = messages[1:] -%}
307
+ {%- endif -%}
308
+
309
+ {# ----- RENDER TOOLS ----- #}
310
+ {%- if tools -%}
311
+ {%- set tools_content = (
312
+ render_tool_namespace('functions', tools)
313
+ + "\n\n"
314
+ ) -%}
315
+ {{- render_role_message({'role': 'function descriptions', 'content': tools_content}) -}}
316
+ {%- endif -%}
317
+
318
+ {# ----- MAIN MESSAGE LOOP ----- #}
319
+ {%- for message in messages -%}
320
+
321
+ {# ----- TOOL MESSAGE -------#}
322
+ {%- if message['role'] == 'tool' -%}
323
+ {{- render_role_message(message, 'function result') -}}
324
+
325
+
326
+ {# ----- ASSISTANT MESSAGE ----- #}
327
+ {%- elif message['role'] == 'assistant' -%}
328
+
329
+ {# ----- FUNCTION CALL PART CHECKING: SINGLE CALL SETUP ----- #}
330
+ {%- if message.tool_calls is defined and message.tool_calls -%}
331
+ {%- set function_call = message.tool_calls[0] -%}
332
+ {%- else -%}
333
+ {%- set function_call = None -%}
334
+ {%- endif -%}
335
+
336
+ {# ----- MAIN ASSISTANT RENDERING ----- #}
337
+
338
+ {{- render_role_message({'role': 'assistant', 'content': message.content}) -}}
339
+ {%- if function_call -%}
340
+ {{- render_function_call({'role': 'function call', 'content': function_call}) -}}
341
+ {%- endif -%}
342
+
343
+
344
+ {# ----- OTHER MESSAGES ----- #}
345
+ {%- else -%}
346
+ {{- render_role_message(message) -}}
347
+ {%- endif -%}
348
+
349
+ {# ----- ADDING GENERATION PROMPT ----- #}
350
+
351
+ {%- if loop.last and add_generation_prompt and message['role'] != 'assistant' -%}
352
+ {{- 'assistant' + add_tokens.role_sep -}}
353
+ {%- endif -%}
354
+
355
+ {%- endfor -%}
models/templates/GigaChat3.1-10B-A1.8B.jinja ADDED
@@ -0,0 +1,339 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {#--------TOOL RENDERING FUNCTIONS---------#}
2
+
3
+ {#---------------------------------------------------------------
4
+ Converts JSON Schema (dict) to a TypeScript type definition
5
+ ----------------------------------------------------------------#}
6
+ {%- macro json_schema_to_typescript(schema, indent="") -%}
7
+ {%- set ADDITIONAL_JSON_KEYS = ['format', 'maxItems', 'maximum', 'minItems', 'minimum', 'pattern'] -%}
8
+ {%- set ty = schema.get("type") -%}
9
+
10
+ {# ---------------- OBJECT ---------------- #}
11
+ {%- if ty == "object" -%}
12
+ {{- "{\n" -}}
13
+
14
+ {# Start building property list #}
15
+ {%- set props = schema.get("properties", {}) -%}
16
+ {%- set required = schema.get("required", []) -%}
17
+ {%- set has_additional_props = schema.get("additionalProperties") is defined -%}
18
+ {%- set additional_props_type = none -%}
19
+ {%- if has_additional_props -%}
20
+ {%- if schema.additionalProperties == true -%}
21
+ {%- set additional_props_type = {'type': 'any'} -%}
22
+ {%- elif schema.additionalProperties is mapping -%}
23
+ {%- set additional_props_type = schema.additionalProperties -%}
24
+ {%- endif -%}
25
+ {%- endif -%}
26
+
27
+ {%- for key, val in props.items() -%}
28
+ {# ---------- Description Comments ---------- #}
29
+ {%- if "description" in val -%}
30
+ {%- for line in val['description'].split('\n') -%}
31
+ {%- if line.strip() -%}
32
+ {{- indent + '// ' + line + '\n' -}}
33
+ {%- endif -%}
34
+ {%- endfor -%}
35
+ {%- endif -%}
36
+
37
+ {# ---------- Additional JSON Keys ---------- #}
38
+ {%- for add_key, add_val in val.items() -%}
39
+ {%- if add_key in ADDITIONAL_JSON_KEYS -%}
40
+ {%- if add_val is string -%}
41
+ {{- indent + '// ' + add_key + ': "' + add_val + '"' + '\n' -}}
42
+ {%- else -%}
43
+ {{- indent + '// ' + add_key + ': ' ~ add_val ~ '\n' -}}
44
+ {%- endif -%}
45
+ {%- endif -%}
46
+ {%- endfor -%}
47
+
48
+ {# ---------- Property Definition ---------- #}
49
+ {%- set type_str = json_schema_to_typescript(
50
+ val,
51
+ indent + " "
52
+ ) -%}
53
+
54
+ {{- indent + key + ('' if key in required else '?') + ': ' + type_str + ',' -}}
55
+
56
+ {%- if "default" in val or "defalut_value" in val -%}
57
+ {%- set default = val.get("default", val.get("defalut_value")) -%}
58
+ {%- if default is string -%}
59
+ {{- ' // default: "' + default + '"' -}}
60
+ {%- else -%}
61
+ {{- ' // default: ' ~ default -}}
62
+ {%- endif -%}
63
+ {%- endif -%}
64
+
65
+ {{- "\n" -}}
66
+ {%- endfor -%}
67
+
68
+ {# Handle additionalProperties as index signature #}
69
+ {%- if has_additional_props and additional_props_type is not none -%}
70
+ {%- set additional_type_str = json_schema_to_typescript(
71
+ additional_props_type,
72
+ indent + " "
73
+ ) -%}
74
+ {{- indent + '[key: string]: ' + additional_type_str + '\n' -}}
75
+ {%- endif -%}
76
+
77
+ {{- indent[: (indent|length - " "|length) ] + '}' -}}
78
+
79
+ {# ---------------- STRING ---------------- #}
80
+ {%- elif ty == "string" -%}
81
+ {%- if schema.get("enum") -%}
82
+ {%- set ns = namespace(enum = []) -%}
83
+ {%- for en in schema['enum'] -%}
84
+ {%- set ns.enum = ns.enum + ['"' ~ en ~ '"'] -%}
85
+ {%- endfor -%}
86
+ {{- ns.enum | join(' | ') -}}
87
+ {%- elif schema.get("format", "none") in ['date-time', 'date'] -%}
88
+ {{- 'Date' -}}
89
+ {%- else -%}
90
+ {{- 'string' -}}
91
+ {%- endif -%}
92
+
93
+ {# ---------------- NUMBER / INTEGER ---------------- #}
94
+ {%- elif ty in ["number", "integer"] -%}
95
+ {%- if schema.get("enum") -%}
96
+ {{- schema.enum | join(' | ') -}}
97
+ {%- else -%}
98
+ {{- 'number' -}}
99
+ {%- endif -%}
100
+
101
+ {# ---------------- BOOLEAN ---------------- #}
102
+ {%- elif ty == "boolean" -%}
103
+ {{- 'boolean' -}}
104
+
105
+ {# ---------------- ARRAY ---------------- #}
106
+ {%- elif ty == "array" -%}
107
+ {%- if "items" in schema -%}
108
+ {{- json_schema_to_typescript(schema['items'], indent) + '[]' -}}
109
+ {%- else -%}
110
+ {{- 'Array<any>' -}}
111
+ {%- endif -%}
112
+
113
+ {# ---------------- FALLBACK ---------------- #}
114
+ {%- else -%}
115
+ {{- 'any' -}}
116
+ {%- endif -%}
117
+ {%- endmacro -%}
118
+
119
+ {#---------------------------------------------------------------
120
+ Renders a namespace and its tool definitions in TypeScript style
121
+ ----------------------------------------------------------------#}
122
+
123
+ {%- macro render_tool_namespace(namespace_name, tools) -%}
124
+ {%- set ns = namespace(sections = ['namespace ' ~ namespace_name ~ ' {']) -%}
125
+
126
+ {%- for tool in tools -%}
127
+ {%- if tool.function -%}
128
+ {%- set tool = tool.function -%}
129
+ {%- endif -%}
130
+
131
+ {%- set ns_tool = namespace(content_lines=[]) -%}
132
+
133
+ {# ---------- TOOL DESCRIPTION ---------- #}
134
+ {%- if tool.get('description') -%}
135
+ {%- for line in tool['description'].split('\n') -%}
136
+ {%- if line.strip() -%}
137
+ {%- set ns_tool.content_lines = ns_tool.content_lines + ['// ' ~ line] -%}
138
+ {%- endif -%}
139
+ {%- endfor -%}
140
+ {%- endif -%}
141
+
142
+ {# ---------- TOOL SIGNATURE ---------- #}
143
+ {%- set main_body = "" -%}
144
+ {%- set params = tool.get("parameters") -%}
145
+ {%- if params and params.get("properties") -%}
146
+ {%- set param_type = json_schema_to_typescript(params, " ") -%}
147
+ {%- set main_body = 'type ' ~ tool.name ~ ' = (_: ' ~ param_type ~ ') => ' -%}
148
+ {%- else -%}
149
+ {%- set main_body = 'type ' ~ tool.name ~ ' = () => ' -%}
150
+ {%- endif -%}
151
+
152
+ {# ---------- RETURN TYPE ---------- #}
153
+ {%- set return_params = tool.get("return_parameters") -%}
154
+ {%- if return_params and return_params.get("properties") -%}
155
+ {%- set return_type = json_schema_to_typescript(return_params, " ") -%}
156
+ {%- set main_body = main_body ~ return_type -%}
157
+ {%- else -%}
158
+ {%- set main_body = main_body ~ 'any' -%}
159
+ {%- endif -%}
160
+
161
+ {%- set main_body = main_body ~ ';\n' -%}
162
+
163
+ {%- set ns_tool.content_lines = ns_tool.content_lines + [main_body] -%}
164
+
165
+ {# ---------- ADD TOOL TO SECTIONS ---------- #}
166
+ {%- set ns.sections = ns.sections + [ns_tool.content_lines | join('\n')] -%}
167
+ {%- endfor -%}
168
+
169
+ {%- set ns.sections = ns.sections + ['} // namespace ' ~ namespace_name] -%}
170
+
171
+ {{- ns.sections | join('\n') -}}
172
+ {%- endmacro -%}
173
+
174
+
175
+ {# ----------- MESSAGE RENDERING HELPER FUNCTIONS ------------ #}
176
+
177
+ {%- macro render_function_call(call) -%}
178
+ {%- if call.function -%}
179
+ {%- set call = call.function -%}
180
+ {%- endif -%}
181
+
182
+ {%- set arguments = call['arguments'] -%}
183
+ {%- if arguments is not string -%}
184
+ {%- set arguments = arguments| tojson(ensure_ascii=False) -%}
185
+ {%- endif -%}
186
+
187
+ {{- '{"name": "' ~ call['name'] ~ '", "arguments": ' ~ arguments ~ '}' -}}
188
+ {%- endmacro -%}
189
+
190
+
191
+ {%- macro render_role_message(message, role=None) -%}
192
+ {%- if not role -%}
193
+ {%- set role = message["role"] -%}
194
+ {%- endif -%}
195
+
196
+ {%- set message_content = message['content'] or '' -%}
197
+ {%- if message_content is not string -%}
198
+ {%- set message_content = message_content | tojson(ensure_ascii=False) -%}
199
+ {%- endif -%}
200
+
201
+ {{- role + add_tokens.role_sep + message_content -}}
202
+
203
+ {%- if message.tool_calls is defined and message.tool_calls -%}
204
+ {{- add_tokens.function_call + render_function_call(message.tool_calls[0]) -}}
205
+ {%- endif -%}
206
+
207
+ {{- add_tokens.message_sep -}}
208
+
209
+ {%- endmacro -%}
210
+
211
+
212
+
213
+ {# ----- SPECIAL TOKENS ----- #}
214
+
215
+ {%- set add_tokens = namespace(
216
+ role_sep="<|role_sep|>\n",
217
+ message_sep="<|message_sep|>\n\n",
218
+ function_call="<|function_call|>"
219
+ ) -%}
220
+
221
+ {# ----- DEFAULT DEVSYSTEM ----- #}
222
+
223
+ {%- set DEVSYSTEM -%}
224
+ <role_description>
225
+ Description of the roles available in the dialog.
226
+
227
+ `developer system`
228
+ A message added by Sber before the main dialog. It has the highest priority and sets global, non-overridable conditions (for example, conversation rules, the safety policy, the assistant's overall response style, etc.).
229
+
230
+ `system`
231
+ A system instruction added by developers or by the user, but with a lower priority than `developer system`. It usually describes the assistant's instructions, a specific response style, and other conditions for this particular dialog.
232
+
233
+ `user`
234
+ A message or request from the user. The assistant follows it if it does not conflict with higher-priority instructions (see <instruction_priority>).
235
+
236
+ `user memory`
237
+ A sequence of the most up-to-date long-term facts about the user at the time of their request, presented as a JSON list of strings. Facts are listed in chronological order, meaning newer facts are appended to the end of the sequence. When facts are changed or deleted, records of previous facts remain in the sequence. The assistant saves facts using a function and uses them in accordance with the <memory_guidelines> block below.
238
+
239
+ `added files`
240
+ Metadata about files available for use in the dialog, presented in JSON format. It contains the following keys: id (a unique file identifier), name (file name), type (file type).
241
+
242
+ `assistant`
243
+ The assistant's reply to the user's request. If the system instruction or the user does not set additional rules for `assistant`, this reply must comply with the instructions in the <assistant_guidelines> block below. The list of functions available to call is contained in `function descriptions`. The name of the required function and its arguments will be generated next by the `function call` role. In its replies, the assistant follows the instructions in accordance with <instruction_priority>.
244
+
245
+ `function descriptions`
246
+ Function descriptions in TypeScript format. A function is a special tool (or a set of instructions) that the assistant can call to perform specific actions, computations, or obtain data needed to solve the user's task. Each function description contains blocks with the name, description, and arguments. Sometimes the description contains separate blocks with return parameters and usage examples that illustrate the correct call and arguments.
247
+
248
+ `function call`
249
+ The function that `assistant` calls based on the dialog context, and its arguments. The function is invoked in strict accordance with the instructions in the <function_usage> block.
250
+
251
+ `function result`
252
+ The result of the last function call.
253
+ </role_description>
254
+
255
+ <available_modalities>
256
+ The assistant can work with the following modalities: text, available functions.
257
+ </available_modalities>
258
+
259
+ <instruction_priority>
260
+ If instructions from different roles conflict within the dialog context, observe the following priorities:
261
+ `developer system` > `system` > `user` > `function descriptions` > `function result` > `user memory`
262
+ </instruction_priority>
263
+
264
+ <function_usage>
265
+ Basic instructions for working with functions.
266
+
267
+ Only call those functions that are described in `function descriptions`.
268
+
269
+ Call available functions when, according to their description, such a call will help provide a more complete and/or accurate answer to the user's request. Fill in function arguments using information from the dialog context. If a function could help answer the request but a required argument is missing from the context, ask the user for the missing data before calling the function. If a necessary function is unavailable or an error occurs, briefly inform the user and, if possible, suggest an alternative.
270
+ </function_usage>
271
+
272
+ <memory_guidelines>
273
+ Rules for using facts in long-term memory:
274
+
275
+ If there is no message under the `user memory` role in the dialog, this is equivalent to the absence of long-term facts about the user in memory. In that case, information about the user is limited to the current dialog, and no new facts should be saved.
276
+ </memory_guidelines>
277
+
278
+ <assistant_guidelines>
279
+ You are a helpful assistant.
280
+
281
+ # Instructions
282
+ - Strictly follow the instruction priority.
283
+ - Maintain a logical chain of reasoning when answering the user's question.
284
+ - For complex questions (for example, STEM), try to answer in detail unless the system message or dialog context limits the response length.
285
+ - Be helpful, truthful, and avoid unsafe or prohibited content in your responses.
286
+ - Try to reply in the language in which the user asked their question.
287
+ </assistant_guidelines>
288
+
289
+ A dialog will follow below.
290
+ The dialog may include various roles described in the <role_description> block.
291
+ Each turn begins with the role name and a special token that marks the end of the role's full name, and ends with a special end-of-turn token.
292
+ Your task is to continue the dialog from the last specified role in accordance with the dialog context.
293
+ {%- endset -%}
294
+
295
+
296
+ {#- ---------------------- RENDERING STARTS HERE ---------------------- -#}
297
+
298
+
299
+ {# ----- RENDER BOS TOKEN ----- #}
300
+ {{- bos_token -}}
301
+
302
+
303
+ {# ----- RENDER DEVSYSTEM ----- #}
304
+ {{- render_role_message({"role": "developer system", "content": DEVSYSTEM}) -}}
305
+
306
+ {# ----- RENDER SYSTEM IF PRESENT ----- #}
307
+ {%- if messages and messages[0]['role'] == 'system' -%}
308
+ {{- render_role_message(messages[0]) -}}
309
+ {%- set messages = messages[1:] -%}
310
+ {%- endif -%}
311
+
312
+ {# ----- RENDER TOOLS ----- #}
313
+ {%- if tools -%}
314
+ {%- set tools_content = (
315
+ render_tool_namespace('functions', tools)
316
+ + "\n\n"
317
+ ) -%}
318
+ {{- render_role_message({'role': 'function descriptions', 'content': tools_content}) -}}
319
+ {%- endif -%}
320
+
321
+ {# ----- MAIN MESSAGE LOOP ----- #}
322
+ {%- for message in messages -%}
323
+
324
+ {# ----- TOOL MESSAGE -------#}
325
+ {%- if message['role'] == 'tool' -%}
326
+ {{- render_role_message(message, 'function result') -}}
327
+
328
+ {# ----- OTHER MESSAGES ----- #}
329
+ {%- else -%}
330
+ {{- render_role_message(message) -}}
331
+ {%- endif -%}
332
+
333
+ {# ----- ADDING GENERATION PROMPT ----- #}
334
+
335
+ {%- if loop.last and add_generation_prompt and message['role'] != 'assistant' -%}
336
+ {{- 'assistant' + add_tokens.role_sep -}}
337
+ {%- endif -%}
338
+
339
+ {%- endfor -%}
models/templates/HuggingFaceTB-SmolLM3-3B.jinja ADDED
@@ -0,0 +1,61 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {#- Copyright 2025-present the Unsloth team. All rights reserved. #}
2
+ {#- Licensed under the Apache License, Version 2.0 (the "License") #}
3
+ {#- Edits made by Unsloth to make it work for most inference engines #}
4
+ {# ───── defaults ───── #}
5
+ {%- if enable_thinking is not defined -%}
6
+ {%- set enable_thinking = true -%}
7
+ {%- endif -%}
8
+ {# ───── reasoning mode ───── #}
9
+ {%- if enable_thinking -%}
10
+ {%- set reasoning_mode = "/think" -%}
11
+ {%- else -%}
12
+ {%- set reasoning_mode = "/no_think" -%}
13
+ {%- endif -%}
14
+ {# ───── header (system message) ───── #}
15
+ {{- "<|im_start|>system\n" -}}
16
+ {%- if messages[0].role == "system" -%}
17
+ {%- set system_message = messages[0].content -%}
18
+ {%- if "/no_think" in system_message -%}
19
+ {%- set reasoning_mode = "/no_think" -%}
20
+ {%- elif "/think" in system_message -%}
21
+ {%- set reasoning_mode = "/think" -%}
22
+ {%- endif -%}
23
+ {%- set custom_instructions = system_message.replace("/no_think", "") -%}
24
+ {%- set custom_instructions = custom_instructions.replace("/think", "") -%}
25
+ {%- set custom_instructions = custom_instructions.rstrip() -%}
26
+ {%- endif -%}
27
+ {{- "## Metadata\n\n" -}}
28
+ {{- "Knowledge Cutoff Date: June 2025\n" -}}
29
+ {{- "Reasoning Mode: " + reasoning_mode + "\n\n" -}}
30
+ {{- "## Custom Instructions\n\n" -}}
31
+ {%- if custom_instructions -%}
32
+ {{- custom_instructions + "\n\n" -}}
33
+ {%- elif reasoning_mode == "/think" -%}
34
+ {{- "You are a helpful AI assistant named SmolLM, trained by Hugging Face. Your role as an assistant involves thoroughly exploring questions through a systematic thinking process before providing the final precise and accurate solutions. This requires engaging in a comprehensive cycle of analysis, summarizing, exploration, reassessment, reflection, backtracking, and iteration to develop well-considered thinking process. Please structure your response into two main sections: Thought and Solution using the specified format: <think> Thought section </think> Solution section. In the Thought section, detail your reasoning process in steps. Each step should include detailed considerations such as analysing questions, summarizing relevant findings, brainstorming new ideas, verifying the accuracy of the current steps, refining any errors, and revisiting previous steps. In the Solution section, based on various attempts, explorations, and reflections from the Thought section, systematically present the final solution that you deem correct. The Solution section should be logical, accurate, and concise and detail necessary steps needed to reach the conclusion.\n\n" -}}
35
+ {%- else -%}
36
+ {{- "You are a helpful AI assistant named SmolLM, trained by Hugging Face.\n\n" -}}
37
+ {%- endif -%}
38
+ {{- "<|im_end|>\n" -}}
39
+ {# ───── main loop ───── #}
40
+ {%- for message in messages -%}
41
+ {%- set content = message.content if message.content is string else "" -%}
42
+ {%- if message.role == "user" -%}
43
+ {{ "<|im_start|>" + message.role + "\n" + content + "<|im_end|>\n" }}
44
+ {%- elif message.role == "assistant" -%}
45
+ {%- if reasoning_mode == "/think" -%}
46
+ {{ "<|im_start|>assistant\n" + content.lstrip("\n") + "<|im_end|>\n" }}
47
+ {%- else -%}
48
+ {{ "<|im_start|>assistant\n" + "<think>\n\n</think>\n" + content.lstrip("\n") + "<|im_end|>\n" }}
49
+ {%- endif -%}
50
+ {%- elif message.role == "tool" -%}
51
+ {{ "<|im_start|>" + "user\n" + content + "<|im_end|>\n" }}
52
+ {%- endif -%}
53
+ {%- endfor -%}
54
+ {# ───── generation prompt ───── #}
55
+ {%- if add_generation_prompt -%}
56
+ {%- if reasoning_mode == "/think" -%}
57
+ {{ "<|im_start|>assistant\n" }}
58
+ {%- else -%}
59
+ {{ "<|im_start|>assistant\n" + "<think>\n\n</think>\n" }}
60
+ {%- endif -%}
61
+ {%- endif -%}
models/templates/Kimi-K2-Instruct.jinja ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {% macro render_content(msg) -%}
2
+ {%- set c = msg.get('content') -%}
3
+ {%- if c is string -%}
4
+ {{ c }}
5
+ {%- elif c is not none -%}
6
+ {% for content in c -%}
7
+ {% if content['type'] == 'image' or 'image' in content or 'image_url' in content -%}
8
+ <|media_start|>image<|media_content|><|media_pad|><|media_end|>
9
+ {% else -%}
10
+ {{ content['text'] }}
11
+ {%- endif -%}
12
+ {%- endfor -%}
13
+ {%- endif -%}
14
+ {%- endmacro %}
15
+
16
+ {%- set tool_response_queue = namespace(ids=[]) -%}
17
+ {%- set tool_call_counter = namespace(value=0) -%}
18
+
19
+ {%- if tools -%}
20
+ <|im_system|>tool_declare<|im_middle|>{{ tools | tojson }}<|im_end|>
21
+ {%- endif -%}
22
+ {% for message in messages %}
23
+ {%- if loop.first and messages[0]['role'] != 'system' -%}
24
+ <|im_system|>system<|im_middle|>You are Kimi, an AI assistant created by Moonshot AI.<|im_end|>
25
+ {% endif %}
26
+
27
+ {%- set role_name = message.get('name') or message['role'] -%}
28
+ {%- if message['role'] == 'user' -%}
29
+ <|im_user|>{{role_name}}<|im_middle|>
30
+ {%- elif message['role'] == 'assistant' -%}
31
+ <|im_assistant|>{{role_name}}<|im_middle|>
32
+ {%- else -%}
33
+ <|im_system|>{{role_name}}<|im_middle|>
34
+ {%- endif -%}
35
+
36
+ {%- if message['role'] == 'assistant' and message.get('tool_calls') -%}
37
+ {{render_content(message)}}<|tool_calls_section_begin|>
38
+ {%- for tool_call in message['tool_calls'] -%}
39
+ {%- set formatted_id = 'functions.' + tool_call['function']['name'] + ':' + (tool_call_counter.value | string) -%}
40
+ {%- set tool_call_counter.value = tool_call_counter.value + 1 -%}
41
+ {%- set _ = tool_response_queue.ids.append(formatted_id) -%}
42
+ <|tool_call_begin|>{{ formatted_id }}<|tool_call_argument_begin|>{% if tool_call['function']['arguments'] is string %}{{ tool_call['function']['arguments'] }}{% else %}{{ tool_call['function']['arguments'] | tojson }}{% endif %}<|tool_call_end|>
43
+ {%- endfor -%}
44
+ <|tool_calls_section_end|>
45
+ {%- elif message['role'] == 'tool' -%}
46
+ {%- if tool_response_queue.ids -%}
47
+ {%- set tool_call_id = tool_response_queue.ids.pop(0) -%}
48
+ {%- else -%}
49
+ {%- set tool_call_id = 'functions.' + message.get('name', 'unknown') + ':' + (tool_call_counter.value | string) -%}
50
+ {%- endif -%}
51
+ ## Return of {{ tool_call_id }}
52
+ {{render_content(message)}}
53
+ {%- elif message['content'] is not none -%}
54
+ {{render_content(message)}}
55
+ {%- endif -%}
56
+ <|im_end|>
57
+ {%- endfor -%}
58
+ {%- if add_generation_prompt -%}
59
+ <|im_assistant|>assistant<|im_middle|>
60
+ {%- endif -%}
models/templates/Kimi-K2-Thinking.jinja ADDED
@@ -0,0 +1,108 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- macro render_content(msg) -%}
2
+ {%- set c = msg.get('content') -%}
3
+ {%- if c is string -%}
4
+ {{ c }}
5
+ {%- elif c is not none -%}
6
+ {% for content in c -%}
7
+ {% if content['type'] == 'image' or 'image' in content or 'image_url' in content -%}
8
+ <|media_start|>image<|media_content|><|media_pad|><|media_end|>
9
+ {% else -%}
10
+ {{ content['text'] }}
11
+ {%- endif -%}
12
+ {%- endfor -%}
13
+ {%- endif -%}
14
+ {%- endmacro -%}
15
+
16
+ {% macro set_roles(message) -%}
17
+ {%- set role_name = message.get('name') or message['role'] -%}
18
+ {%- if message['role'] == 'user' -%}
19
+ <|im_user|>{{role_name}}<|im_middle|>
20
+ {%- elif message['role'] == 'assistant' -%}
21
+ <|im_assistant|>{{role_name}}<|im_middle|>
22
+ {%- else -%}
23
+ <|im_system|>{{role_name}}<|im_middle|>
24
+ {%- endif -%}
25
+ {%- endmacro -%}
26
+
27
+ {%- set tool_response_queue = namespace(ids=[]) -%}
28
+ {%- set tool_call_counter = namespace(value=0) -%}
29
+
30
+ {%- macro render_toolcalls(message) -%}
31
+ <|tool_calls_section_begin|>
32
+ {%- for tool_call in message['tool_calls'] -%}
33
+ {%- set formatted_id = 'functions.' + tool_call['function']['name'] + ':' + (tool_call_counter.value | string) -%}
34
+ {%- set tool_call_counter.value = tool_call_counter.value + 1 -%}
35
+ {%- set _ = tool_response_queue.ids.append(formatted_id) -%}
36
+ <|tool_call_begin|>{{ formatted_id }}<|tool_call_argument_begin|>{% if tool_call['function']['arguments'] is string %}{{ tool_call['function']['arguments'] }}{% else %}{{ tool_call['function']['arguments'] | tojson }}{% endif %}<|tool_call_end|>
37
+ {%- endfor -%}
38
+ <|tool_calls_section_end|>
39
+ {%- endmacro -%}
40
+
41
+
42
+ {# Find last non-tool-call assisitant message #}
43
+ {%- set ns = namespace(last_non_tool_call_assistant_msg=-1) -%}
44
+ {%- for idx in range(messages|length-1, -1, -1) -%}
45
+ {%- if messages[idx]['role'] == 'assistant' and not messages[idx].get('tool_calls') -%}
46
+ {%- set ns.last_non_tool_call_assistant_msg = idx -%}
47
+ {%- endif -%}
48
+ {%- endfor -%}
49
+
50
+ {# split all messages into history & suffix, reasoning_content in suffix should be reserved.#}
51
+ {%- set hist_msgs = messages[:ns.last_non_tool_call_assistant_msg+1] -%}
52
+ {%- set suffix_msgs = messages[ns.last_non_tool_call_assistant_msg+1:] -%}
53
+
54
+ {%- if tools -%}
55
+ <|im_system|>tool_declare<|im_middle|>{{ tools | tojson }}<|im_end|>
56
+ {%- endif -%}
57
+
58
+ {%- if messages|length == 0 or messages[0]['role'] != 'system' -%}
59
+ <|im_system|>system<|im_middle|>You are Kimi, an AI assistant created by Moonshot AI.<|im_end|>
60
+ {%- endif -%}
61
+
62
+ {%- for message in hist_msgs -%}
63
+ {{set_roles(message)}}
64
+ {%- if message['role'] == 'assistant' -%}
65
+ <think></think>{{render_content(message)}}
66
+ {%- if message.get('tool_calls') -%}
67
+ {{render_toolcalls(message)}}
68
+ {%- endif -%}
69
+ {%- elif message['role'] == 'tool' -%}
70
+ {%- if tool_response_queue.ids -%}
71
+ {%- set tool_call_id = tool_response_queue.ids.pop(0) -%}
72
+ {%- else -%}
73
+ {%- set tool_call_id = 'functions.' + message.get('name', 'unknown') + ':' + (tool_call_counter.value | string) -%}
74
+ {%- endif -%}
75
+ ## Return of {{ tool_call_id }}
76
+ {{render_content(message)}}
77
+ {%- elif message['content'] is not none -%}
78
+ {{render_content(message)}}
79
+ {%- endif -%}
80
+ <|im_end|>
81
+ {%- endfor -%}
82
+
83
+ {%- for message in suffix_msgs -%}
84
+ {{set_roles(message)}}
85
+ {%- if message['role'] == 'assistant' -%}
86
+ {%- set rc = message.get('reasoning_content', '') -%}
87
+ <think>{{rc}}</think>{{render_content(message)}}
88
+ {%- if message.get('tool_calls') -%}
89
+ {{render_toolcalls(message)}}
90
+ {%- endif -%}
91
+ {%- elif message['role'] == 'tool' -%}
92
+ {%- if tool_response_queue.ids -%}
93
+ {%- set tool_call_id = tool_response_queue.ids.pop(0) -%}
94
+ {%- else -%}
95
+ {%- set tool_call_id = 'functions.' + message.get('name', 'unknown') + ':' + (tool_call_counter.value | string) -%}
96
+ {%- endif -%}
97
+ ## Return of {{ tool_call_id }}
98
+ {{render_content(message)}}
99
+ {%- elif message['content'] is not none -%}
100
+ {{render_content(message)}}
101
+ {%- endif -%}
102
+ <|im_end|>
103
+ {%- endfor -%}
104
+
105
+
106
+ {%- if add_generation_prompt -%}
107
+ <|im_assistant|>assistant<|im_middle|>
108
+ {%- endif -%}
models/templates/Kimi-K3.jinja ADDED
@@ -0,0 +1,324 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- macro escape_attr(value) -%}
2
+ {{- value|string|replace('&', '&amp;')|replace('"', '&quot;') -}}
3
+ {%- endmacro -%}
4
+
5
+ {%- macro open_tag(tag, attrs=[]) -%}
6
+ {{- '<|open|>' + tag -}}
7
+ {%- for attr in attrs -%}
8
+ {{- ' ' + attr[0] + '="' -}}{{- escape_attr(attr[1]) -}}{{- '"' -}}
9
+ {%- endfor -%}
10
+ {{- '<|sep|>' -}}
11
+ {%- endmacro -%}
12
+
13
+ {%- macro close_tag(tag) -%}
14
+ {{- '<|close|>' + tag + '<|sep|>' -}}
15
+ {%- endmacro -%}
16
+
17
+ {%- macro next_image(state) -%}
18
+ {%- if image_prompts is defined and image_prompts is not none -%}
19
+ {%- if state.image_index >= image_prompts|length -%}
20
+ {{- raise_exception('More image placeholders than image prompts.') -}}
21
+ {%- endif -%}
22
+ {{- image_prompts[state.image_index] -}}
23
+ {%- set state.image_index = state.image_index + 1 -%}
24
+ {%- else -%}
25
+ {{- '<|kimi_image_placeholder|>' -}}
26
+ {%- endif -%}
27
+ {%- endmacro -%}
28
+
29
+ {%- macro render_text(text, state) -%}
30
+ {%- set text = text|string -%}
31
+ {%- if image_prompts is defined and image_prompts is not none and '<|kimi_image_placeholder|>' in text -%}
32
+ {%- set parts = text.split('<|kimi_image_placeholder|>') -%}
33
+ {%- for part in parts -%}
34
+ {{- part -}}
35
+ {%- if not loop.last -%}{{- next_image(state) -}}{%- endif -%}
36
+ {%- endfor -%}
37
+ {%- else -%}
38
+ {{- text -}}
39
+ {%- endif -%}
40
+ {%- endmacro -%}
41
+
42
+ {%- macro render_content(content, state) -%}
43
+ {%- if content is string -%}
44
+ {{- render_text(content, state) -}}
45
+ {%- elif content is not none and content is defined -%}
46
+ {%- for part in content -%}
47
+ {%- if part.type in ['image', 'image_url'] -%}
48
+ {{- next_image(state) -}}
49
+ {%- else -%}
50
+ {{- render_text(part.text, state) -}}
51
+ {%- endif -%}
52
+ {%- endfor -%}
53
+ {%- endif -%}
54
+ {%- endmacro -%}
55
+
56
+ {%- macro internal_system_message(message_type, body) -%}
57
+ {{- open_tag('message', [('role', 'system'), ('type', message_type)]) -}}
58
+ {{- body|trim -}}
59
+ {{- close_tag('message') -}}
60
+ {{- '<|end_of_msg|>' -}}
61
+ {%- endmacro -%}
62
+
63
+ {%- macro json_sorted(value) -%}
64
+ {#- tojson has no sort_keys, so sort each mapping level with dictsort to match the
65
+ reference implementation. Array order is kept as-is. -#}
66
+ {%- if value is mapping -%}
67
+ {{- '{' -}}
68
+ {%- for key, item in value|dictsort -%}
69
+ {%- if not loop.first -%}{{- ',' -}}{%- endif -%}
70
+ {{- key|tojson(ensure_ascii=false) -}}{{- ':' -}}{{- json_sorted(item) -}}
71
+ {%- endfor -%}
72
+ {{- '}' -}}
73
+ {%- elif value is string or value is number or value is boolean or value is none -%}
74
+ {{- value|tojson(ensure_ascii=false) -}}
75
+ {%- else -%}
76
+ {{- '[' -}}
77
+ {%- for item in value -%}
78
+ {%- if not loop.first -%}{{- ',' -}}{%- endif -%}
79
+ {{- json_sorted(item) -}}
80
+ {%- endfor -%}
81
+ {{- ']' -}}
82
+ {%- endif -%}
83
+ {%- endmacro -%}
84
+
85
+ {%- macro render_tool_declare(tool_list, dynamic=false) -%}
86
+ {{- open_tag('message', [('role', 'system'), ('type', 'tool-declare')]) -}}
87
+ {%- if dynamic -%}
88
+ {{- '## New Tools Available\nThe system dynamically extends the toolset via lazy-loading.\nYou have access to all existing and extended tools.\nHere are the specs for the extended tools.\n\n```json\n' -}}
89
+ {%- else -%}
90
+ {{- '# Tools\nHere are the available tools, described in JSONSchema.\n\n```json\n' -}}
91
+ {%- endif -%}
92
+ {{- json_sorted(tool_list) -}}
93
+ {{- '\n```' -}}
94
+ {{- close_tag('message') -}}
95
+ {{- '<|end_of_msg|>' -}}
96
+ {%- endmacro -%}
97
+
98
+ {%- macro xtml_type(value) -%}
99
+ {%- if value is boolean -%}boolean
100
+ {%- elif value is none -%}null
101
+ {%- elif value is number -%}number
102
+ {%- elif value is string -%}string
103
+ {%- elif value is mapping -%}object
104
+ {%- else -%}array
105
+ {%- endif -%}
106
+ {%- endmacro -%}
107
+
108
+ {%- macro xtml_value(value) -%}
109
+ {%- if value is string -%}
110
+ {{- value -}}
111
+ {%- else -%}
112
+ {{- value|tojson(ensure_ascii=false) -}}
113
+ {%- endif -%}
114
+ {%- endmacro -%}
115
+
116
+ {%- macro render_assistant(message, state) -%}
117
+ {%- if thinking -%}
118
+ {%- set reasoning_content = message.get('reasoning_content') or message.get('reasoning') -%}
119
+ {{- open_tag('think') -}}
120
+ {%- if reasoning_content is not none and reasoning_content|string|trim -%}
121
+ {{- render_text(reasoning_content, state) -}}
122
+ {%- endif -%}
123
+ {{- close_tag('think') -}}
124
+ {%- endif -%}
125
+ {{- open_tag('response') -}}
126
+ {{- render_content(message.get('content'), state) -}}
127
+ {{- close_tag('response') -}}
128
+ {%- set tool_calls = message.get('tool_calls') -%}
129
+ {%- if tool_calls -%}
130
+ {{- open_tag('tools') -}}
131
+ {%- for tool_call in tool_calls -%}
132
+ {%- if tool_call is not mapping -%}
133
+ {{- raise_exception('Kimi K3 tool calls must be mappings.') -}}
134
+ {%- endif -%}
135
+ {%- set fn = tool_call.function if tool_call.function is defined and tool_call.function is mapping else tool_call -%}
136
+ {%- if fn.get('name') is none -%}
137
+ {{- raise_exception('Kimi K3 tool calls require a function name.') -}}
138
+ {%- endif -%}
139
+ {{- open_tag('call', [('tool', fn.name), ('index', loop.index)]) -}}
140
+ {%- set arguments = fn.get('arguments', {}) -%}
141
+ {%- set json_block = fn.get('_xtml_json_block') -%}
142
+ {%- if json_block is not none -%}
143
+ {{- open_tag('json', [('type', 'object')]) -}}
144
+ {{- render_text(json_block, state) -}}
145
+ {{- close_tag('json') -}}
146
+ {%- elif arguments is mapping -%}
147
+ {%- for key, value in arguments.items() -%}
148
+ {{- open_tag('argument', [('key', key), ('type', xtml_type(value))]) -}}
149
+ {{- render_text(xtml_value(value), state) -}}
150
+ {{- close_tag('argument') -}}
151
+ {%- endfor -%}
152
+ {%- elif arguments is string and arguments|trim -%}
153
+ {{- open_tag('json', [('type', 'object')]) -}}
154
+ {{- render_text(arguments, state) -}}
155
+ {{- close_tag('json') -}}
156
+ {%- elif arguments is not none and arguments is not string -%}
157
+ {{- raise_exception('Kimi K3 tool call arguments must be a mapping or a JSON object string.') -}}
158
+ {%- endif -%}
159
+ {{- close_tag('call') -}}
160
+ {%- endfor -%}
161
+ {{- close_tag('tools') -}}
162
+ {%- endif -%}
163
+ {%- endmacro -%}
164
+
165
+ {%- macro render_tool_message(message, state, resolved_name=none) -%}
166
+ {%- set state.tool_index = state.tool_index + 1 -%}
167
+ {%- if resolved_name is not none -%}
168
+ {%- set tool_name = resolved_name -%}
169
+ {%- elif 'tool' in message -%}
170
+ {%- set tool_name = message.get('tool') -%}
171
+ {%- else -%}
172
+ {%- set tool_name = message.get('name') -%}
173
+ {%- endif -%}
174
+ {%- if tool_name is none and state.tool_calls is not none and state.tool_index <= state.tool_calls|length -%}
175
+ {%- set fallback_call = state.tool_calls[state.tool_index - 1] -%}
176
+ {%- set fallback_fn = fallback_call.function if fallback_call.function is defined and fallback_call.function is mapping else fallback_call -%}
177
+ {%- set tool_name = fallback_fn.name -%}
178
+ {%- endif -%}
179
+ {%- if tool_name is none -%}
180
+ {{- raise_exception('Kimi K3 tool messages need a resolvable tool name: carry `tool`/`name`, or match a preceding assistant tool_call by order.') -}}
181
+ {%- endif -%}
182
+ {{- open_tag('message', [('role', 'tool'), ('tool', tool_name), ('index', state.tool_index)]) -}}
183
+ {{- render_content(message.get('content'), state) -}}
184
+ {{- close_tag('message') -}}
185
+ {{- '<|end_of_msg|>' -}}
186
+ {%- endmacro -%}
187
+
188
+ {%- if thinking is undefined -%}
189
+ {%- set thinking = true -%}
190
+ {%- endif -%}
191
+ {%- if thinking_effort is undefined -%}
192
+ {%- set thinking_effort = 'max' -%}
193
+ {%- endif -%}
194
+ {%- if thinking and thinking_effort is not none and thinking_effort not in ['low', 'high', 'max'] -%}
195
+ {{- raise_exception('Unsupported thinking_effort=' + thinking_effort|string + '; supported values are low, high, and max.') -}}
196
+ {%- endif -%}
197
+
198
+ {%- set state = namespace(image_index=0, tool_calls=none, tool_index=0, response_schema=none) -%}
199
+
200
+ {%- if tools is defined and tools -%}
201
+ {{- render_tool_declare(tools) -}}
202
+ {%- endif -%}
203
+
204
+ {%- if thinking and thinking_effort in ['low', 'high', 'max'] -%}
205
+ {{- internal_system_message(
206
+ 'thinking-effort',
207
+ '`thinking_effort` guides on how much to think in your thinking channel (not including the response channel), supported values include `low`, `medium`, `high`, and `max`.\nNow the system is invoked with `thinking_effort=' + thinking_effort|string + '`.'
208
+ ) -}}
209
+ {%- endif -%}
210
+
211
+ {%- for message in messages -%}
212
+ {%- if message is mapping -%}
213
+ {%- if 'role' not in message -%}
214
+ {{- raise_exception('Kimi K3 messages require a role.') -}}
215
+ {%- elif message.role == 'user' -%}
216
+ {%- set attrs = [('role', 'user')] -%}
217
+ {%- if message.get('name') -%}{%- set attrs = attrs + [('name', message.name)] -%}{%- endif -%}
218
+ {{- open_tag('message', attrs) -}}
219
+ {{- render_content(message.get('content'), state) -}}
220
+ {{- close_tag('message') -}}
221
+ {{- '<|end_of_msg|>' -}}
222
+ {%- elif message.role == 'system' and message.get('tools') -%}
223
+ {{- render_tool_declare(message.tools, dynamic=true) -}}
224
+ {%- elif message.role == 'system' -%}
225
+ {%- set attrs = [('role', 'system')] -%}
226
+ {%- if message.get('name') -%}{%- set attrs = attrs + [('name', message.name)] -%}{%- endif -%}
227
+ {{- open_tag('message', attrs) -}}
228
+ {{- render_content(message.get('content'), state) -}}
229
+ {{- close_tag('message') -}}
230
+ {{- '<|end_of_msg|>' -}}
231
+ {%- elif message.role == 'assistant' -%}
232
+ {%- set state.tool_calls = message.get('tool_calls') -%}
233
+ {%- set state.tool_index = 0 -%}
234
+ {%- set attrs = [('role', 'assistant')] -%}
235
+ {%- if message.get('name') -%}{%- set attrs = attrs + [('name', message.name)] -%}{%- endif -%}
236
+ {{- open_tag('message', attrs) -}}
237
+ {{- render_assistant(message, state) -}}
238
+ {{- close_tag('message') -}}
239
+ {{- '<|end_of_msg|>' -}}
240
+ {%- elif message.role == 'tool' and (loop.first or messages[loop.index0 - 1].role != 'tool') -%}
241
+ {%- set run = namespace(tool_messages=[], resolved_count=0) -%}
242
+ {%- for candidate in messages[loop.index0:] -%}
243
+ {%- if candidate is not mapping or candidate.role != 'tool' -%}{%- break -%}{%- endif -%}
244
+ {%- set run.tool_messages = run.tool_messages + [candidate] -%}
245
+ {%- set call_id = candidate.get('tool_call_id', candidate.get('id')) -%}
246
+ {%- set match = namespace(found=false) -%}
247
+ {%- if call_id is not none and state.tool_calls is not none -%}
248
+ {%- for tool_call in state.tool_calls -%}
249
+ {%- if not match.found and tool_call is mapping and tool_call.get('id') is not none and tool_call.get('id')|string == call_id|string -%}
250
+ {%- set match.found = true -%}
251
+ {%- endif -%}
252
+ {%- endfor -%}
253
+ {%- endif -%}
254
+ {%- if match.found -%}{%- set run.resolved_count = run.resolved_count + 1 -%}{%- endif -%}
255
+ {%- endfor -%}
256
+ {%- if run.tool_messages|length > 0 and run.resolved_count == run.tool_messages|length -%}
257
+ {%- set emitted = namespace(ids=[]) -%}
258
+ {%- for tool_call in state.tool_calls -%}
259
+ {%- if tool_call is mapping and tool_call.get('id') is not none and tool_call.get('id')|string not in emitted.ids -%}
260
+ {%- set emitted.ids = emitted.ids + [tool_call.get('id')|string] -%}
261
+ {%- set fn = tool_call.function if tool_call.function is defined and tool_call.function is mapping else tool_call -%}
262
+ {%- for tool_message in run.tool_messages -%}
263
+ {%- set result_id = tool_message.get('tool_call_id', tool_message.get('id')) -%}
264
+ {%- if result_id is not none and result_id|string == tool_call.get('id')|string -%}
265
+ {{- render_tool_message(tool_message, state, fn.get('name')) -}}
266
+ {%- endif -%}
267
+ {%- endfor -%}
268
+ {%- endif -%}
269
+ {%- endfor -%}
270
+ {%- else -%}
271
+ {%- for tool_message in run.tool_messages -%}
272
+ {{- render_tool_message(tool_message, state) -}}
273
+ {%- endfor -%}
274
+ {%- endif -%}
275
+ {%- endif -%}
276
+ {%- endif -%}
277
+ {%- endfor -%}
278
+
279
+ {%- if tool_choice is defined and tool_choice == 'required' -%}
280
+ {{- internal_system_message('tool-choice', 'The system is invoked with `tool_choice=required`.\nYou MUST call tools in the next message.') -}}
281
+ {%- elif tool_choice is defined and tool_choice == 'none' -%}
282
+ {{- internal_system_message('tool-choice', 'The system is invoked with `tool_choice=none`.\nYou MUST NOT call any tools in the next message.') -}}
283
+ {%- endif -%}
284
+
285
+ {%- if response_schema is defined -%}
286
+ {%- set state.response_schema = response_schema -%}
287
+ {%- elif response_format is defined and response_format is mapping and response_format.get('json_schema') is not none -%}
288
+ {%- set schema_wrapper = response_format.get('json_schema') -%}
289
+ {%- if schema_wrapper is mapping and 'schema' in schema_wrapper -%}
290
+ {%- set state.response_schema = schema_wrapper.get('schema') -%}
291
+ {%- elif schema_wrapper is mapping and 'json_schema' in schema_wrapper -%}
292
+ {%- set state.response_schema = schema_wrapper.get('json_schema') -%}
293
+ {%- else -%}
294
+ {%- set state.response_schema = schema_wrapper -%}
295
+ {%- endif -%}
296
+ {%- endif -%}
297
+
298
+ {%- set response_format_type = none -%}
299
+ {%- if response_format is defined and response_format is mapping -%}
300
+ {%- set response_format_type = response_format.get('type') -%}
301
+ {%- elif response_format is defined -%}
302
+ {%- set response_format_type = response_format -%}
303
+ {%- endif -%}
304
+ {%- if response_format_type == 'json_object' -%}
305
+ {{- internal_system_message(
306
+ 'response-format',
307
+ 'The system is invoked with `response_format=json_object`.\nYour response must be raw JSON data without markdown code blocks (```json) or any additional formatting.'
308
+ ) -}}
309
+ {%- elif response_format_type == 'json_schema' -%}
310
+ {{- internal_system_message(
311
+ 'response-format',
312
+ 'The system is invoked with `response_format=json_schema`.\nYour response must be raw JSON data without markdown code blocks (```json) or any additional formatting.\nThe JSON data must match the following schema:\n```json\n' + json_sorted(state.response_schema) + '\n```'
313
+ ) -}}
314
+ {%- endif -%}
315
+
316
+ {%- if add_generation_prompt -%}
317
+ {{- open_tag('message', [('role', 'assistant')]) -}}
318
+ {{- open_tag('think' if thinking else 'response') -}}
319
+ {%- endif -%}
320
+
321
+ {%- if image_prompts is defined and image_prompts is not none and state.image_index != image_prompts|length -%}
322
+ {{- raise_exception('image prompt count ' + image_prompts|length|string + ' != consumed placeholder count ' + state.image_index|string) -}}
323
+ {%- endif -%}
324
+