Add build-info files, CI workflows, build scripts and chat templates missed by staging glob
Browse filesThis view is limited to 50 files because it contains too many changes. See raw diff
- .github/workflows/build-3rd-party.yml +57 -0
- .github/workflows/build-and-test-snapdragon.yml +148 -0
- .github/workflows/build-android.yml +149 -0
- .github/workflows/build-apple.yml +287 -0
- .github/workflows/build-cache.yml +118 -0
- .github/workflows/build-cann.yml +104 -0
- .github/workflows/build-cmake-pkg.yml +53 -0
- .github/workflows/build-cpu.yml +235 -0
- .github/workflows/build-cross.yml +317 -0
- .github/workflows/build-cuda-ubuntu.yml +193 -0
- .github/workflows/build-cuda-windows.yml +183 -0
- .github/workflows/build-ibm.yml +150 -0
- .github/workflows/build-msys.yml +70 -0
- .github/workflows/build-opencl.yml +92 -0
- .github/workflows/build-openvino.yml +178 -0
- .github/workflows/build-riscv.yml +188 -0
- .github/workflows/build-sanitize.yml +108 -0
- .github/workflows/build-self-hosted.yml +409 -0
- .github/workflows/build-sycl.yml +171 -0
- .github/workflows/build-virtgpu.yml +50 -0
- .github/workflows/build-vulkan.yml +232 -0
- .github/workflows/build-wasm.yml +110 -0
- .github/workflows/build-webgpu.yml +195 -0
- cmake/build-info.cmake +48 -0
- common/build-info.cpp.in +35 -0
- common/build-info.h +13 -0
- docs/build-riscv64-spacemit.md +110 -0
- docs/build-s390x.md +275 -0
- docs/build.md +844 -0
- examples/llama.android/app/build.gradle.kts +58 -0
- examples/llama.android/build.gradle.kts +6 -0
- examples/llama.android/lib/build.gradle.kts +79 -0
- examples/sycl/build.sh +53 -0
- examples/test-cmake/build-install.sh +19 -0
- examples/test-cmake/build.sh +7 -0
- models/templates/Apertus-8B-Instruct.jinja +327 -0
- models/templates/Apriel-1.6-15b-Thinker-fixed.jinja +171 -0
- models/templates/Bielik-11B-v3.0-Instruct.jinja +77 -0
- models/templates/ByteDance-Seed-OSS.jinja +171 -0
- models/templates/Cohere2MoE.jinja +263 -0
- models/templates/CohereForAI-c4ai-command-r-plus-tool_use.jinja +202 -0
- models/templates/CohereForAI-c4ai-command-r7b-12-2024-tool_use.jinja +156 -0
- models/templates/GLM-4.6.jinja +106 -0
- models/templates/GLM-4.7-Flash.jinja +86 -0
- models/templates/GigaChat3-10B-A1.8B.jinja +355 -0
- models/templates/GigaChat3.1-10B-A1.8B.jinja +339 -0
- models/templates/HuggingFaceTB-SmolLM3-3B.jinja +61 -0
- models/templates/Kimi-K2-Instruct.jinja +60 -0
- models/templates/Kimi-K2-Thinking.jinja +108 -0
- models/templates/Kimi-K3.jinja +324 -0
.github/workflows/build-3rd-party.yml
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: CI (3rd-party)
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
workflow_dispatch: # allows manual triggering
|
| 5 |
+
push:
|
| 6 |
+
branches:
|
| 7 |
+
- master
|
| 8 |
+
paths: [
|
| 9 |
+
'.github/workflows/build-3rd-party.yml',
|
| 10 |
+
'**/CMakeLists.txt',
|
| 11 |
+
'**/.cmake',
|
| 12 |
+
'**/*.h',
|
| 13 |
+
'**/*.hpp',
|
| 14 |
+
'**/*.c',
|
| 15 |
+
'**/*.cpp'
|
| 16 |
+
]
|
| 17 |
+
|
| 18 |
+
concurrency:
|
| 19 |
+
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
| 20 |
+
cancel-in-progress: true
|
| 21 |
+
|
| 22 |
+
env:
|
| 23 |
+
GGML_NLOOP: 3
|
| 24 |
+
GGML_N_THREADS: 1
|
| 25 |
+
LLAMA_ARG_LOG_COLORS: 1
|
| 26 |
+
LLAMA_ARG_LOG_PREFIX: 1
|
| 27 |
+
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
| 28 |
+
|
| 29 |
+
jobs:
|
| 30 |
+
ubuntu-24-llguidance:
|
| 31 |
+
runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
|
| 32 |
+
|
| 33 |
+
steps:
|
| 34 |
+
- name: Clone
|
| 35 |
+
id: checkout
|
| 36 |
+
uses: actions/checkout@v6
|
| 37 |
+
|
| 38 |
+
- name: Dependencies
|
| 39 |
+
id: depends
|
| 40 |
+
run: |
|
| 41 |
+
sudo apt-get update
|
| 42 |
+
sudo apt-get install build-essential libssl-dev
|
| 43 |
+
|
| 44 |
+
- name: Build
|
| 45 |
+
id: cmake_build
|
| 46 |
+
run: |
|
| 47 |
+
cmake -B build \
|
| 48 |
+
-DLLAMA_FATAL_WARNINGS=ON \
|
| 49 |
+
-DLLAMA_LLGUIDANCE=ON
|
| 50 |
+
cmake --build build --config Release -j $(nproc)
|
| 51 |
+
|
| 52 |
+
- name: Test
|
| 53 |
+
id: cmake_test
|
| 54 |
+
run: |
|
| 55 |
+
cd build
|
| 56 |
+
ctest -L main --verbose --timeout 900
|
| 57 |
+
|
.github/workflows/build-and-test-snapdragon.yml
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: CI (snapdragon)
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
workflow_dispatch:
|
| 5 |
+
push:
|
| 6 |
+
branches:
|
| 7 |
+
- master
|
| 8 |
+
paths:
|
| 9 |
+
- '.github/workflows/build-and-test-snapdragon.yml'
|
| 10 |
+
- 'ggml/include/ggml-hexagon.h'
|
| 11 |
+
- 'ggml/src/ggml-hexagon/**'
|
| 12 |
+
- 'docs/backend/snapdragon/**'
|
| 13 |
+
- 'scripts/snapdragon/**'
|
| 14 |
+
- 'CMakePresets.json'
|
| 15 |
+
|
| 16 |
+
pull_request:
|
| 17 |
+
types: [opened, synchronize, reopened]
|
| 18 |
+
paths:
|
| 19 |
+
- '.github/workflows/build-and-test-snapdragon.yml'
|
| 20 |
+
- 'ggml/include/ggml-hexagon.h'
|
| 21 |
+
- 'ggml/src/ggml-hexagon/**'
|
| 22 |
+
- 'docs/backend/snapdragon/**'
|
| 23 |
+
- 'scripts/snapdragon/**'
|
| 24 |
+
- 'CMakePresets.json'
|
| 25 |
+
|
| 26 |
+
concurrency:
|
| 27 |
+
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
| 28 |
+
cancel-in-progress: true
|
| 29 |
+
|
| 30 |
+
jobs:
|
| 31 |
+
android-ndk-snapdragon:
|
| 32 |
+
runs-on: ubuntu-latest
|
| 33 |
+
container:
|
| 34 |
+
image: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.7'
|
| 35 |
+
defaults:
|
| 36 |
+
run:
|
| 37 |
+
shell: bash
|
| 38 |
+
|
| 39 |
+
steps:
|
| 40 |
+
- name: Clone
|
| 41 |
+
uses: actions/checkout@v6
|
| 42 |
+
with:
|
| 43 |
+
fetch-depth: 0
|
| 44 |
+
lfs: false
|
| 45 |
+
|
| 46 |
+
- name: Build Llama.CPP for Snapdragon Android
|
| 47 |
+
id: build_llama_cpp_snapdragon_android
|
| 48 |
+
run: |
|
| 49 |
+
cp docs/backend/snapdragon/CMakeUserPresets.json .
|
| 50 |
+
cmake --preset arm64-android-snapdragon-release -B build
|
| 51 |
+
cmake --build build
|
| 52 |
+
cmake --install build --prefix pkg-snapdragon/llama.cpp
|
| 53 |
+
|
| 54 |
+
- name: Upload Llama.CPP Snapdragon Android Build Artifact
|
| 55 |
+
if: ${{ always() && steps.build_llama_cpp_snapdragon_android.outcome == 'success' }}
|
| 56 |
+
uses: actions/upload-artifact@v6
|
| 57 |
+
with:
|
| 58 |
+
name: llama-cpp-android-arm64-snapdragon
|
| 59 |
+
path: pkg-snapdragon/llama.cpp
|
| 60 |
+
|
| 61 |
+
linux-iot-snapdragon:
|
| 62 |
+
runs-on: ubuntu-latest
|
| 63 |
+
container:
|
| 64 |
+
image: 'ghcr.io/snapdragon-toolchain/arm64-linux:v0.7'
|
| 65 |
+
defaults:
|
| 66 |
+
run:
|
| 67 |
+
shell: bash
|
| 68 |
+
|
| 69 |
+
steps:
|
| 70 |
+
- name: Clone
|
| 71 |
+
uses: actions/checkout@v6
|
| 72 |
+
with:
|
| 73 |
+
fetch-depth: 0
|
| 74 |
+
lfs: false
|
| 75 |
+
|
| 76 |
+
- name: Build Llama.CPP for Snapdragon Linux IoT
|
| 77 |
+
id: build_llama_cpp_snapdragon_linux
|
| 78 |
+
run: |
|
| 79 |
+
cp docs/backend/snapdragon/CMakeUserPresets.json .
|
| 80 |
+
cmake --preset arm64-linux-snapdragon-release -B build-snapdragon -DGGML_OPENCL=ON
|
| 81 |
+
cmake --build build-snapdragon -j $(nproc)
|
| 82 |
+
cmake --install build-snapdragon --prefix pkg-snapdragon/llama.cpp
|
| 83 |
+
|
| 84 |
+
- name: Upload Llama.CPP Snapdragon Linux IoT Build Artifact
|
| 85 |
+
if: ${{ always() && steps.build_llama_cpp_snapdragon_linux.outcome == 'success' }}
|
| 86 |
+
uses: actions/upload-artifact@v6
|
| 87 |
+
with:
|
| 88 |
+
name: llama-cpp-linux-arm64-snapdragon
|
| 89 |
+
path: pkg-snapdragon/llama.cpp
|
| 90 |
+
|
| 91 |
+
test-snapdragon-qdc:
|
| 92 |
+
name: Test on QDC Device (${{ matrix.device }})
|
| 93 |
+
needs: [android-ndk-snapdragon, linux-iot-snapdragon]
|
| 94 |
+
runs-on: ubuntu-24.04-arm
|
| 95 |
+
timeout-minutes: 90
|
| 96 |
+
strategy:
|
| 97 |
+
fail-fast: false
|
| 98 |
+
matrix:
|
| 99 |
+
device: [SM8750, SM8850, QCS9075M]
|
| 100 |
+
|
| 101 |
+
steps:
|
| 102 |
+
- name: Checkout
|
| 103 |
+
uses: actions/checkout@v6
|
| 104 |
+
|
| 105 |
+
- name: Download build artifact
|
| 106 |
+
uses: actions/download-artifact@v7
|
| 107 |
+
with:
|
| 108 |
+
name: ${{ startsWith(matrix.device, 'QCS') && 'llama-cpp-linux-arm64-snapdragon' || 'llama-cpp-android-arm64-snapdragon' }}
|
| 109 |
+
path: pkg-snapdragon/llama.cpp
|
| 110 |
+
|
| 111 |
+
- name: Set up Python
|
| 112 |
+
uses: actions/setup-python@v6
|
| 113 |
+
with:
|
| 114 |
+
python-version: '3.x'
|
| 115 |
+
cache: pip
|
| 116 |
+
|
| 117 |
+
- name: Install system dependencies
|
| 118 |
+
run: |
|
| 119 |
+
sudo apt-get update
|
| 120 |
+
sudo apt-get install -y curl unzip
|
| 121 |
+
|
| 122 |
+
- name: Install QDC SDK wheel
|
| 123 |
+
run: |
|
| 124 |
+
curl -fSL -o qdc_sdk.zip https://softwarecenter.qualcomm.com/api/download/software/tools/Qualcomm_Device_Cloud_SDK/All/0.2.3/qualcomm_device_cloud_sdk-0.2.3.zip
|
| 125 |
+
unzip qdc_sdk.zip -d qdc_sdk
|
| 126 |
+
pip install qdc_sdk/qualcomm_device_cloud_sdk-0.2.3-py3-none-any.whl
|
| 127 |
+
|
| 128 |
+
- name: Check QDC API key
|
| 129 |
+
id: check_secret
|
| 130 |
+
env:
|
| 131 |
+
QDC_API_KEY: ${{ secrets.QDC_API_KEY }}
|
| 132 |
+
run: echo "has-qdc-key=${{ env.QDC_API_KEY != '' }}" >> "$GITHUB_OUTPUT"
|
| 133 |
+
|
| 134 |
+
- name: Run QDC tests (${{ matrix.device }})
|
| 135 |
+
if: steps.check_secret.outputs.has-qdc-key == 'true'
|
| 136 |
+
run: |
|
| 137 |
+
python scripts/snapdragon/qdc/run_qdc_jobs.py \
|
| 138 |
+
--test all \
|
| 139 |
+
--pkg-dir pkg-snapdragon/llama.cpp \
|
| 140 |
+
--model-url "https://huggingface.co/bartowski/Llama-3.2-1B-Instruct-GGUF/resolve/main/Llama-3.2-1B-Instruct-Q4_0.gguf" \
|
| 141 |
+
--device ${{ matrix.device }} \
|
| 142 |
+
${{ startsWith(matrix.device, 'QCS') && '--retries 2 --retry-delay 300' || '' }}
|
| 143 |
+
env:
|
| 144 |
+
QDC_API_KEY: ${{ secrets.QDC_API_KEY }}
|
| 145 |
+
|
| 146 |
+
- name: Cleanup
|
| 147 |
+
if: always()
|
| 148 |
+
run: rm -rf pkg-snapdragon qdc_sdk qdc_sdk.zip
|
.github/workflows/build-android.yml
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: CI (android)
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
workflow_dispatch:
|
| 5 |
+
push:
|
| 6 |
+
branches:
|
| 7 |
+
- master
|
| 8 |
+
paths:
|
| 9 |
+
- '.github/workflows/build-android.yml'
|
| 10 |
+
- '**/CMakeLists.txt'
|
| 11 |
+
- '**/.cmake'
|
| 12 |
+
- '**/*.h'
|
| 13 |
+
- '**/*.hpp'
|
| 14 |
+
- '**/*.c'
|
| 15 |
+
- '**/*.cpp'
|
| 16 |
+
|
| 17 |
+
pull_request:
|
| 18 |
+
types: [opened, synchronize, reopened]
|
| 19 |
+
paths:
|
| 20 |
+
- '.github/workflows/build-android.yml'
|
| 21 |
+
- 'examples/llama.android/**'
|
| 22 |
+
|
| 23 |
+
concurrency:
|
| 24 |
+
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
| 25 |
+
cancel-in-progress: true
|
| 26 |
+
|
| 27 |
+
env:
|
| 28 |
+
GGML_NLOOP: 3
|
| 29 |
+
GGML_N_THREADS: 1
|
| 30 |
+
LLAMA_ARG_LOG_COLORS: 1
|
| 31 |
+
LLAMA_ARG_LOG_PREFIX: 1
|
| 32 |
+
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
| 33 |
+
|
| 34 |
+
jobs:
|
| 35 |
+
default:
|
| 36 |
+
runs-on: ubuntu-latest
|
| 37 |
+
|
| 38 |
+
steps:
|
| 39 |
+
- name: Clone
|
| 40 |
+
uses: actions/checkout@v6
|
| 41 |
+
with:
|
| 42 |
+
fetch-depth: 0
|
| 43 |
+
lfs: false
|
| 44 |
+
|
| 45 |
+
- name: Set up JDK
|
| 46 |
+
uses: actions/setup-java@v5
|
| 47 |
+
with:
|
| 48 |
+
java-version: 17
|
| 49 |
+
distribution: zulu
|
| 50 |
+
|
| 51 |
+
- name: Setup Android SDK
|
| 52 |
+
uses: android-actions/setup-android@40fd30fb8d7440372e1316f5d1809ec01dcd3699 # v4.0.1
|
| 53 |
+
with:
|
| 54 |
+
log-accepted-android-sdk-licenses: false
|
| 55 |
+
|
| 56 |
+
- name: Build
|
| 57 |
+
run: |
|
| 58 |
+
cd examples/llama.android
|
| 59 |
+
./gradlew build --no-daemon
|
| 60 |
+
|
| 61 |
+
ndk:
|
| 62 |
+
runs-on: ubuntu-latest
|
| 63 |
+
container:
|
| 64 |
+
image: 'ghcr.io/snapdragon-toolchain/arm64-android:v0.3'
|
| 65 |
+
defaults:
|
| 66 |
+
run:
|
| 67 |
+
shell: bash
|
| 68 |
+
|
| 69 |
+
steps:
|
| 70 |
+
- name: Clone
|
| 71 |
+
uses: actions/checkout@v6
|
| 72 |
+
with:
|
| 73 |
+
fetch-depth: 0
|
| 74 |
+
lfs: false
|
| 75 |
+
|
| 76 |
+
- name: Dependencies
|
| 77 |
+
run: |
|
| 78 |
+
apt-get update
|
| 79 |
+
apt-get install -y build-essential
|
| 80 |
+
|
| 81 |
+
- name: Build
|
| 82 |
+
id: ndk_build
|
| 83 |
+
run: |
|
| 84 |
+
cmake -D ANDROID_ABI=arm64-v8a -D ANDROID_PLATFORM=android-31 -D CMAKE_TOOLCHAIN_FILE=${ANDROID_NDK_ROOT}/build/cmake/android.toolchain.cmake -D GGML_NATIVE=OFF -DGGML_CPU_ARM_ARCH=armv8.5-a+fp16+i8mm -G Ninja -D LLAMA_OPENSSL=OFF -D GGML_OPENMP=OFF -B build
|
| 85 |
+
cmake --build build
|
| 86 |
+
cmake --install build --prefix pkg-adb/llama.cpp
|
| 87 |
+
|
| 88 |
+
- name: Upload Android Build Artifact
|
| 89 |
+
if: ${{ always() && steps.ndk_build.outcome == 'success' }}
|
| 90 |
+
uses: actions/upload-artifact@v6
|
| 91 |
+
with:
|
| 92 |
+
name: llama-cpp-android-arm64-cpu
|
| 93 |
+
path: pkg-adb/llama.cpp
|
| 94 |
+
|
| 95 |
+
arm64:
|
| 96 |
+
runs-on: ubuntu-latest
|
| 97 |
+
|
| 98 |
+
env:
|
| 99 |
+
NDK_VERSION: "29.0.14206865"
|
| 100 |
+
|
| 101 |
+
steps:
|
| 102 |
+
- name: Clone
|
| 103 |
+
id: checkout
|
| 104 |
+
uses: actions/checkout@v6
|
| 105 |
+
|
| 106 |
+
# note : disabled to spare some cache space (https://github.com/ggml-org/llama.cpp/pull/23789)
|
| 107 |
+
# for some reason, the ccache does not improve the build time in this case
|
| 108 |
+
# example:
|
| 109 |
+
# cache off: https://github.com/ggerganov/tmp2/actions/runs/26534713799/job/78160400831
|
| 110 |
+
# cache on: https://github.com/ggerganov/tmp2/actions/runs/26534713799/job/78224189394
|
| 111 |
+
#
|
| 112 |
+
#- name: ccache
|
| 113 |
+
# uses: ggml-org/ccache-action@v1.2.24
|
| 114 |
+
# with:
|
| 115 |
+
# key: android-ubuntu-arm64
|
| 116 |
+
# evict-old-files: 1d
|
| 117 |
+
# save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
| 118 |
+
|
| 119 |
+
- name: Set up JDK
|
| 120 |
+
uses: actions/setup-java@v5
|
| 121 |
+
with:
|
| 122 |
+
java-version: 17
|
| 123 |
+
distribution: temurin
|
| 124 |
+
|
| 125 |
+
- name: Setup Android SDK
|
| 126 |
+
uses: android-actions/setup-android@40fd30fb8d7440372e1316f5d1809ec01dcd3699 # v4.0.1
|
| 127 |
+
with:
|
| 128 |
+
log-accepted-android-sdk-licenses: false
|
| 129 |
+
|
| 130 |
+
- name: Install NDK
|
| 131 |
+
run: |
|
| 132 |
+
sdkmanager "ndk;${{ env.NDK_VERSION }}"
|
| 133 |
+
echo "ANDROID_NDK=${ANDROID_SDK_ROOT}/ndk/${{ env.NDK_VERSION }}" >> $GITHUB_ENV
|
| 134 |
+
|
| 135 |
+
- name: Build
|
| 136 |
+
id: cmake_build
|
| 137 |
+
run: |
|
| 138 |
+
cmake -B build \
|
| 139 |
+
-DCMAKE_TOOLCHAIN_FILE=${ANDROID_NDK}/build/cmake/android.toolchain.cmake \
|
| 140 |
+
-DANDROID_ABI=arm64-v8a \
|
| 141 |
+
-DANDROID_PLATFORM=android-28 \
|
| 142 |
+
-DLLAMA_FATAL_WARNINGS=ON \
|
| 143 |
+
-DGGML_BACKEND_DL=ON \
|
| 144 |
+
-DGGML_NATIVE=OFF \
|
| 145 |
+
-DGGML_CPU_ALL_VARIANTS=ON \
|
| 146 |
+
-DGGML_OPENMP=OFF \
|
| 147 |
+
-DLLAMA_BUILD_BORINGSSL=ON \
|
| 148 |
+
-DGGML_RPC=ON
|
| 149 |
+
time cmake --build build --config Release -j $(nproc)
|
.github/workflows/build-apple.yml
ADDED
|
@@ -0,0 +1,287 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: CI (apple)
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
workflow_dispatch: # allows manual triggering
|
| 5 |
+
push:
|
| 6 |
+
branches:
|
| 7 |
+
- master
|
| 8 |
+
paths: [
|
| 9 |
+
'.github/workflows/build-apple.yml',
|
| 10 |
+
'**/CMakeLists.txt',
|
| 11 |
+
'**/.cmake',
|
| 12 |
+
'**/*.h',
|
| 13 |
+
'**/*.hpp',
|
| 14 |
+
'**/*.c',
|
| 15 |
+
'**/*.cpp',
|
| 16 |
+
'**/*.swift',
|
| 17 |
+
'**/*.m',
|
| 18 |
+
'**/*.metal'
|
| 19 |
+
]
|
| 20 |
+
|
| 21 |
+
pull_request:
|
| 22 |
+
types: [opened, synchronize, reopened]
|
| 23 |
+
paths: [
|
| 24 |
+
'.github/workflows/build-apple.yml',
|
| 25 |
+
'ggml/src/ggml-metal/**',
|
| 26 |
+
'ggml/src/ggml-rpc/**'
|
| 27 |
+
]
|
| 28 |
+
|
| 29 |
+
concurrency:
|
| 30 |
+
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
| 31 |
+
cancel-in-progress: true
|
| 32 |
+
|
| 33 |
+
env:
|
| 34 |
+
GGML_NLOOP: 3
|
| 35 |
+
GGML_N_THREADS: 1
|
| 36 |
+
LLAMA_ARG_LOG_COLORS: 1
|
| 37 |
+
LLAMA_ARG_LOG_PREFIX: 1
|
| 38 |
+
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
| 39 |
+
|
| 40 |
+
jobs:
|
| 41 |
+
macos-latest-arm64:
|
| 42 |
+
runs-on: macos-latest
|
| 43 |
+
|
| 44 |
+
steps:
|
| 45 |
+
- name: Clone
|
| 46 |
+
id: checkout
|
| 47 |
+
uses: actions/checkout@v6
|
| 48 |
+
|
| 49 |
+
- name: ccache
|
| 50 |
+
uses: ggml-org/ccache-action@v1.2.24
|
| 51 |
+
with:
|
| 52 |
+
key: apple-arm64
|
| 53 |
+
save: false
|
| 54 |
+
|
| 55 |
+
- name: ccache-buckets-restore
|
| 56 |
+
uses: ./.github/actions/ccache-buckets
|
| 57 |
+
env:
|
| 58 |
+
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
| 59 |
+
with:
|
| 60 |
+
key: apple-arm64
|
| 61 |
+
folder: llama.cpp
|
| 62 |
+
hf_bucket: ggml-org/cache
|
| 63 |
+
|
| 64 |
+
- name: Build
|
| 65 |
+
id: cmake_build
|
| 66 |
+
run: |
|
| 67 |
+
sysctl -a
|
| 68 |
+
cmake -B build \
|
| 69 |
+
-DCMAKE_BUILD_RPATH="@loader_path" \
|
| 70 |
+
-DLLAMA_FATAL_WARNINGS=ON \
|
| 71 |
+
-DLLAMA_BUILD_BORINGSSL=ON \
|
| 72 |
+
-DGGML_METAL_EMBED_LIBRARY=OFF \
|
| 73 |
+
-DGGML_METAL_SHADER_DEBUG=ON \
|
| 74 |
+
-DGGML_RPC=ON \
|
| 75 |
+
-DCMAKE_OSX_DEPLOYMENT_TARGET=13.3
|
| 76 |
+
time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
|
| 77 |
+
|
| 78 |
+
- name: ccache-buckets-save
|
| 79 |
+
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
| 80 |
+
uses: ./.github/actions/ccache-buckets
|
| 81 |
+
env:
|
| 82 |
+
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
| 83 |
+
with:
|
| 84 |
+
key: apple-arm64
|
| 85 |
+
folder: llama.cpp
|
| 86 |
+
evict-old-files: 1d
|
| 87 |
+
hf_bucket: ggml-org/cache
|
| 88 |
+
save: true
|
| 89 |
+
|
| 90 |
+
- name: Check for leaks
|
| 91 |
+
run: |
|
| 92 |
+
cmd=(./build/bin/test-thread-safety -hf ggml-org/gemma-3-270m-qat-GGUF -ngl 99 -p "$(printf 'hello %.0s' {1..128})" -n 16 -c 512 -ub 32 -np 2 -t 2 -lv 1)
|
| 93 |
+
leaks -atExit -- "${cmd[@]}"
|
| 94 |
+
# Graphics devices are leaked by Metal in Apple code sometimes, so we ignore those leaks
|
| 95 |
+
OBJC_DEBUG_MISSING_POOLS=YES "${cmd[@]}" 2>&1 | awk '{ print } index($0, "autoreleased with no pool in place") && !/class [a-zA-Z0-9]+Device autoreleased/ { found = 1 } END { exit found }'
|
| 96 |
+
|
| 97 |
+
- name: Test
|
| 98 |
+
id: cmake_test
|
| 99 |
+
run: |
|
| 100 |
+
cd build
|
| 101 |
+
ctest -L main -E "test-llama-archs" --verbose --timeout 900
|
| 102 |
+
|
| 103 |
+
macos-latest-x64:
|
| 104 |
+
runs-on: macos-15-intel
|
| 105 |
+
|
| 106 |
+
steps:
|
| 107 |
+
- name: Clone
|
| 108 |
+
id: checkout
|
| 109 |
+
uses: actions/checkout@v6
|
| 110 |
+
|
| 111 |
+
- name: ccache
|
| 112 |
+
uses: ggml-org/ccache-action@v1.2.24
|
| 113 |
+
with:
|
| 114 |
+
key: apple-x64
|
| 115 |
+
save: false
|
| 116 |
+
|
| 117 |
+
- name: ccache-buckets-restore
|
| 118 |
+
uses: ./.github/actions/ccache-buckets
|
| 119 |
+
env:
|
| 120 |
+
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
| 121 |
+
with:
|
| 122 |
+
key: apple-x64
|
| 123 |
+
folder: llama.cpp
|
| 124 |
+
hf_bucket: ggml-org/cache
|
| 125 |
+
|
| 126 |
+
- name: Build
|
| 127 |
+
id: cmake_build
|
| 128 |
+
run: |
|
| 129 |
+
sysctl -a
|
| 130 |
+
# Metal is disabled due to intermittent failures with Github runners not having a GPU:
|
| 131 |
+
# https://github.com/ggml-org/llama.cpp/actions/runs/8635935781/job/23674807267#step:5:2313
|
| 132 |
+
cmake -B build \
|
| 133 |
+
-DCMAKE_BUILD_RPATH="@loader_path" \
|
| 134 |
+
-DLLAMA_FATAL_WARNINGS=ON \
|
| 135 |
+
-DLLAMA_BUILD_BORINGSSL=ON \
|
| 136 |
+
-DGGML_METAL=OFF \
|
| 137 |
+
-DGGML_RPC=ON \
|
| 138 |
+
-DCMAKE_OSX_DEPLOYMENT_TARGET=13.3
|
| 139 |
+
time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
|
| 140 |
+
|
| 141 |
+
- name: ccache-buckets-save
|
| 142 |
+
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
| 143 |
+
uses: ./.github/actions/ccache-buckets
|
| 144 |
+
env:
|
| 145 |
+
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
| 146 |
+
with:
|
| 147 |
+
key: apple-x64
|
| 148 |
+
folder: llama.cpp
|
| 149 |
+
evict-old-files: 1d
|
| 150 |
+
hf_bucket: ggml-org/cache
|
| 151 |
+
save: true
|
| 152 |
+
|
| 153 |
+
- name: Test
|
| 154 |
+
id: cmake_test
|
| 155 |
+
run: |
|
| 156 |
+
cd build
|
| 157 |
+
ctest -L main --verbose --timeout 900
|
| 158 |
+
|
| 159 |
+
macos-latest-ios-xcode:
|
| 160 |
+
runs-on: macos-latest
|
| 161 |
+
|
| 162 |
+
steps:
|
| 163 |
+
- name: Checkout code
|
| 164 |
+
uses: actions/checkout@v6
|
| 165 |
+
|
| 166 |
+
- name: Setup Xcode
|
| 167 |
+
uses: ggml-org/setup-xcode@v1
|
| 168 |
+
with:
|
| 169 |
+
xcode-version: latest-stable
|
| 170 |
+
|
| 171 |
+
- name: Build
|
| 172 |
+
id: cmake_build
|
| 173 |
+
run: |
|
| 174 |
+
sysctl -a
|
| 175 |
+
cmake -B build -G Xcode \
|
| 176 |
+
-DGGML_METAL_EMBED_LIBRARY=ON \
|
| 177 |
+
-DLLAMA_OPENSSL=OFF \
|
| 178 |
+
-DLLAMA_BUILD_APP=OFF \
|
| 179 |
+
-DLLAMA_BUILD_EXAMPLES=OFF \
|
| 180 |
+
-DLLAMA_BUILD_TOOLS=OFF \
|
| 181 |
+
-DLLAMA_BUILD_TESTS=OFF \
|
| 182 |
+
-DLLAMA_BUILD_SERVER=OFF \
|
| 183 |
+
-DCMAKE_SYSTEM_NAME=iOS \
|
| 184 |
+
-DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
|
| 185 |
+
-DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
|
| 186 |
+
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
|
| 187 |
+
|
| 188 |
+
- name: xcodebuild for swift package
|
| 189 |
+
id: xcodebuild
|
| 190 |
+
run: |
|
| 191 |
+
./build-xcframework.sh
|
| 192 |
+
|
| 193 |
+
- name: Upload xcframework artifact
|
| 194 |
+
uses: actions/upload-artifact@v6
|
| 195 |
+
with:
|
| 196 |
+
name: llama-xcframework
|
| 197 |
+
path: build-apple/llama.xcframework/
|
| 198 |
+
retention-days: 1
|
| 199 |
+
|
| 200 |
+
- name: Build Xcode project
|
| 201 |
+
run: |
|
| 202 |
+
xcodebuild -downloadPlatform iOS
|
| 203 |
+
xcodebuild -project examples/llama.swiftui/llama.swiftui.xcodeproj -scheme llama.swiftui -sdk iphoneos CODE_SIGNING_REQUIRED=NO CODE_SIGN_IDENTITY= -destination 'generic/platform=iOS' FRAMEWORK_FOLDER_PATH=./build-ios build
|
| 204 |
+
|
| 205 |
+
macos-latest-tvos:
|
| 206 |
+
runs-on: macos-latest
|
| 207 |
+
|
| 208 |
+
steps:
|
| 209 |
+
- name: Clone
|
| 210 |
+
id: checkout
|
| 211 |
+
uses: actions/checkout@v6
|
| 212 |
+
|
| 213 |
+
- name: Build
|
| 214 |
+
id: cmake_build
|
| 215 |
+
run: |
|
| 216 |
+
sysctl -a
|
| 217 |
+
cmake -B build -G Xcode \
|
| 218 |
+
-DGGML_METAL_EMBED_LIBRARY=ON \
|
| 219 |
+
-DLLAMA_BUILD_COMMON=OFF \
|
| 220 |
+
-DLLAMA_BUILD_APP=OFF \
|
| 221 |
+
-DLLAMA_BUILD_EXAMPLES=OFF \
|
| 222 |
+
-DLLAMA_BUILD_TOOLS=OFF \
|
| 223 |
+
-DLLAMA_BUILD_TESTS=OFF \
|
| 224 |
+
-DLLAMA_BUILD_SERVER=OFF \
|
| 225 |
+
-DCMAKE_SYSTEM_NAME=tvOS \
|
| 226 |
+
-DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
|
| 227 |
+
-DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
|
| 228 |
+
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
|
| 229 |
+
|
| 230 |
+
macos-latest-visionos:
|
| 231 |
+
runs-on: macos-latest
|
| 232 |
+
|
| 233 |
+
steps:
|
| 234 |
+
- name: Clone
|
| 235 |
+
id: checkout
|
| 236 |
+
uses: actions/checkout@v6
|
| 237 |
+
|
| 238 |
+
- name: Build
|
| 239 |
+
id: cmake_build
|
| 240 |
+
run: |
|
| 241 |
+
sysctl -a
|
| 242 |
+
cmake -B build -G Xcode \
|
| 243 |
+
-DGGML_METAL_EMBED_LIBRARY=ON \
|
| 244 |
+
-DLLAMA_BUILD_COMMON=OFF \
|
| 245 |
+
-DLLAMA_BUILD_APP=OFF \
|
| 246 |
+
-DLLAMA_BUILD_EXAMPLES=OFF \
|
| 247 |
+
-DLLAMA_BUILD_TOOLS=OFF \
|
| 248 |
+
-DLLAMA_BUILD_TESTS=OFF \
|
| 249 |
+
-DLLAMA_BUILD_SERVER=OFF \
|
| 250 |
+
-DCMAKE_SYSTEM_NAME=visionOS \
|
| 251 |
+
-DCMAKE_OSX_DEPLOYMENT_TARGET=1.0 \
|
| 252 |
+
-DCMAKE_XCODE_ATTRIBUTE_DEVELOPMENT_TEAM=ggml
|
| 253 |
+
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu) -- CODE_SIGNING_ALLOWED=NO
|
| 254 |
+
|
| 255 |
+
macos-latest-swift:
|
| 256 |
+
runs-on: macos-latest
|
| 257 |
+
needs: macos-latest-ios-xcode
|
| 258 |
+
|
| 259 |
+
strategy:
|
| 260 |
+
matrix:
|
| 261 |
+
destination: ['generic/platform=macOS', 'generic/platform=iOS', 'generic/platform=tvOS']
|
| 262 |
+
|
| 263 |
+
steps:
|
| 264 |
+
- name: Clone
|
| 265 |
+
id: checkout
|
| 266 |
+
uses: actions/checkout@v6
|
| 267 |
+
|
| 268 |
+
- name: Download xcframework artifact
|
| 269 |
+
uses: actions/download-artifact@v7
|
| 270 |
+
with:
|
| 271 |
+
name: llama-xcframework
|
| 272 |
+
path: build-apple/llama.xcframework/
|
| 273 |
+
|
| 274 |
+
- name: Build llama.cpp with CMake
|
| 275 |
+
id: cmake_build
|
| 276 |
+
run: |
|
| 277 |
+
sysctl -a
|
| 278 |
+
cmake -B build -G Xcode \
|
| 279 |
+
-DGGML_METAL_EMBED_LIBRARY=ON \
|
| 280 |
+
-DLLAMA_OPENSSL=OFF \
|
| 281 |
+
-DLLAMA_BUILD_APP=OFF \
|
| 282 |
+
-DLLAMA_BUILD_EXAMPLES=OFF \
|
| 283 |
+
-DLLAMA_BUILD_TOOLS=OFF \
|
| 284 |
+
-DLLAMA_BUILD_TESTS=OFF \
|
| 285 |
+
-DLLAMA_BUILD_SERVER=OFF \
|
| 286 |
+
-DCMAKE_OSX_ARCHITECTURES="arm64;x86_64"
|
| 287 |
+
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
|
.github/workflows/build-cache.yml
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: Build Actions Cache
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
workflow_dispatch: # allows manual triggering
|
| 5 |
+
schedule:
|
| 6 |
+
- cron: '0 * * * *'
|
| 7 |
+
|
| 8 |
+
concurrency:
|
| 9 |
+
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
| 10 |
+
cancel-in-progress: true
|
| 11 |
+
|
| 12 |
+
jobs:
|
| 13 |
+
#ubuntu-24-spacemit-cache:
|
| 14 |
+
# runs-on: ubuntu-24.04
|
| 15 |
+
|
| 16 |
+
# env:
|
| 17 |
+
# # Make sure this is in sync with build-linux-cross.yml
|
| 18 |
+
# SPACEMIT_IME_TOOLCHAIN_VERSION: "1.1.2"
|
| 19 |
+
|
| 20 |
+
# steps:
|
| 21 |
+
# - name: Clone
|
| 22 |
+
# id: checkout
|
| 23 |
+
# uses: actions/checkout@v6
|
| 24 |
+
|
| 25 |
+
# - name: Setup Cache
|
| 26 |
+
# uses: actions/cache@v5
|
| 27 |
+
# id: cache-toolchain
|
| 28 |
+
# with:
|
| 29 |
+
# path: ./spacemit_toolchain
|
| 30 |
+
# key: cache-gha-spacemit-ime-toolchain-v${{ env.SPACEMIT_IME_TOOLCHAIN_VERSION }}-${{ runner.os }}
|
| 31 |
+
|
| 32 |
+
# - name: Setup SpacemiT Toolchain
|
| 33 |
+
# if: steps.cache-toolchain.outputs.cache-hit != 'true'
|
| 34 |
+
# uses: ./.github/actions/linux-setup-spacemit
|
| 35 |
+
# with:
|
| 36 |
+
# path: ./spacemit_toolchain
|
| 37 |
+
# version: ${{ env.SPACEMIT_IME_TOOLCHAIN_VERSION }}
|
| 38 |
+
|
| 39 |
+
ubuntu-24-openvino-cache:
|
| 40 |
+
runs-on: ubuntu-24.04
|
| 41 |
+
|
| 42 |
+
env:
|
| 43 |
+
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
|
| 44 |
+
OPENVINO_VERSION_MAJOR: "2026.3.1"
|
| 45 |
+
OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
|
| 46 |
+
|
| 47 |
+
steps:
|
| 48 |
+
- name: Clone
|
| 49 |
+
id: checkout
|
| 50 |
+
uses: actions/checkout@v6
|
| 51 |
+
|
| 52 |
+
- name: Setup Cache
|
| 53 |
+
uses: actions/cache@v5
|
| 54 |
+
id: cache-openvino
|
| 55 |
+
with:
|
| 56 |
+
path: ./openvino_toolkit
|
| 57 |
+
key: cache-gha-openvino-toolkit-v${{ env.OPENVINO_VERSION_FULL }}-${{ runner.os }}
|
| 58 |
+
|
| 59 |
+
- name: Setup OpenVINO Toolkit
|
| 60 |
+
if: steps.cache-openvino.outputs.cache-hit != 'true'
|
| 61 |
+
uses: ./.github/actions/linux-setup-openvino
|
| 62 |
+
with:
|
| 63 |
+
path: ./openvino_toolkit
|
| 64 |
+
version_major: ${{ env.OPENVINO_VERSION_MAJOR }}
|
| 65 |
+
version_full: ${{ env.OPENVINO_VERSION_FULL }}
|
| 66 |
+
|
| 67 |
+
windows-2022-openvino-cache:
|
| 68 |
+
runs-on: windows-2022
|
| 69 |
+
|
| 70 |
+
env:
|
| 71 |
+
# Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
|
| 72 |
+
OPENVINO_VERSION_MAJOR: "2026.3.1"
|
| 73 |
+
OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
|
| 74 |
+
|
| 75 |
+
steps:
|
| 76 |
+
- name: Clone
|
| 77 |
+
id: checkout
|
| 78 |
+
uses: actions/checkout@v6
|
| 79 |
+
|
| 80 |
+
- name: Setup Cache
|
| 81 |
+
uses: actions/cache@v5
|
| 82 |
+
id: cache-openvino
|
| 83 |
+
with:
|
| 84 |
+
path: ./openvino_toolkit
|
| 85 |
+
key: cache-gha-openvino-toolkit-v${{ env.OPENVINO_VERSION_FULL }}-${{ runner.os }}
|
| 86 |
+
|
| 87 |
+
- name: Setup OpenVINO Toolkit
|
| 88 |
+
if: steps.cache-openvino.outputs.cache-hit != 'true'
|
| 89 |
+
uses: ./.github/actions/windows-setup-openvino
|
| 90 |
+
with:
|
| 91 |
+
path: ./openvino_toolkit
|
| 92 |
+
version_major: ${{ env.OPENVINO_VERSION_MAJOR }}
|
| 93 |
+
version_full: ${{ env.OPENVINO_VERSION_FULL }}
|
| 94 |
+
|
| 95 |
+
# windows-2022-rocm-cache:
|
| 96 |
+
# runs-on: windows-2022
|
| 97 |
+
|
| 98 |
+
# env:
|
| 99 |
+
# # Make sure this is in sync with release.yml and build-cuda-windows.yml
|
| 100 |
+
# ROCM_VERSION: "7.14.0"
|
| 101 |
+
|
| 102 |
+
# steps:
|
| 103 |
+
# - name: Clone
|
| 104 |
+
# id: checkout
|
| 105 |
+
# uses: actions/checkout@v6
|
| 106 |
+
|
| 107 |
+
# - name: Setup Cache
|
| 108 |
+
# uses: actions/cache@v5
|
| 109 |
+
# id: cache-rocm
|
| 110 |
+
# with:
|
| 111 |
+
# path: C:\TheRock\build
|
| 112 |
+
# key: rocm-wheels-${{ env.ROCM_VERSION }}-multi-arch-${{ runner.os }}
|
| 113 |
+
|
| 114 |
+
# - name: Setup ROCm
|
| 115 |
+
# if: steps.cache-rocm.outputs.cache-hit != 'true'
|
| 116 |
+
# uses: ./.github/actions/windows-setup-rocm
|
| 117 |
+
# with:
|
| 118 |
+
# version: ${{ env.ROCM_VERSION }}
|
.github/workflows/build-cann.yml
ADDED
|
@@ -0,0 +1,104 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: CI (cann)
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
workflow_dispatch: # allows manual triggering
|
| 5 |
+
push:
|
| 6 |
+
branches:
|
| 7 |
+
- master
|
| 8 |
+
paths: [
|
| 9 |
+
'.github/workflows/build-cann.yml',
|
| 10 |
+
'**/CMakeLists.txt',
|
| 11 |
+
'**/.cmake',
|
| 12 |
+
'**/*.h',
|
| 13 |
+
'**/*.hpp',
|
| 14 |
+
'**/*.c',
|
| 15 |
+
'**/*.cpp'
|
| 16 |
+
]
|
| 17 |
+
|
| 18 |
+
pull_request:
|
| 19 |
+
types: [opened, synchronize, reopened]
|
| 20 |
+
paths: [
|
| 21 |
+
'.github/workflows/build-cann.yml',
|
| 22 |
+
'ggml/src/ggml-cann/**'
|
| 23 |
+
]
|
| 24 |
+
|
| 25 |
+
concurrency:
|
| 26 |
+
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
| 27 |
+
cancel-in-progress: true
|
| 28 |
+
|
| 29 |
+
env:
|
| 30 |
+
GGML_NLOOP: 3
|
| 31 |
+
GGML_N_THREADS: 1
|
| 32 |
+
LLAMA_ARG_LOG_COLORS: 1
|
| 33 |
+
LLAMA_ARG_LOG_PREFIX: 1
|
| 34 |
+
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
| 35 |
+
|
| 36 |
+
jobs:
|
| 37 |
+
# TODO: this build is disabled to save Github Actions resources (https://github.com/ggml-org/llama.cpp/pull/23705)
|
| 38 |
+
# in order to enable it again, we have to provision dedicated runners to run it
|
| 39 |
+
# openEuler-latest-cann:
|
| 40 |
+
# defaults:
|
| 41 |
+
# run:
|
| 42 |
+
# shell: bash -el {0}
|
| 43 |
+
# strategy:
|
| 44 |
+
# matrix:
|
| 45 |
+
# arch: [x86, aarch64]
|
| 46 |
+
# chip_type: ['910b', '310p']
|
| 47 |
+
# build: ['Release']
|
| 48 |
+
# use_acl_graph: ['on', 'off']
|
| 49 |
+
# exclude:
|
| 50 |
+
# # 310P does not support USE_ACL_GRAPH=on
|
| 51 |
+
# - chip_type: '310p'
|
| 52 |
+
# use_acl_graph: 'on'
|
| 53 |
+
# runs-on: ${{ matrix.arch == 'aarch64' && 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
|
| 54 |
+
# steps:
|
| 55 |
+
# - name: Checkout
|
| 56 |
+
# uses: actions/checkout@v6
|
| 57 |
+
# with:
|
| 58 |
+
# fetch-depth: 0
|
| 59 |
+
#
|
| 60 |
+
# - name: Free up disk space
|
| 61 |
+
# uses: ggml-org/free-disk-space@v1.3.1
|
| 62 |
+
# with:
|
| 63 |
+
# tool-cache: true
|
| 64 |
+
#
|
| 65 |
+
# - name: Set container image
|
| 66 |
+
# id: cann-image
|
| 67 |
+
# run: |
|
| 68 |
+
# image="ascendai/cann:${{ matrix.chip_type == '910b' && '8.5.0-910b-openeuler24.03-py3.11' || '8.5.0-310p-openeuler24.03-py3.11' }}"
|
| 69 |
+
# echo "image=${image}" >> "${GITHUB_OUTPUT}"
|
| 70 |
+
#
|
| 71 |
+
# - name: Pull container image
|
| 72 |
+
# run: docker pull "${{ steps.cann-image.outputs.image }}"
|
| 73 |
+
#
|
| 74 |
+
# - name: Build
|
| 75 |
+
# env:
|
| 76 |
+
# BUILD_TYPE: ${{ matrix.build }}
|
| 77 |
+
# SOC_TYPE: ascend${{ matrix.chip_type }}
|
| 78 |
+
# USE_ACL_GRAPH: ${{ matrix.use_acl_graph }}
|
| 79 |
+
# run: |
|
| 80 |
+
# HOST_UID=$(id -u)
|
| 81 |
+
# HOST_GID=$(id -g)
|
| 82 |
+
#
|
| 83 |
+
# docker run --rm \
|
| 84 |
+
# -v "${PWD}:/workspace" \
|
| 85 |
+
# -w /workspace \
|
| 86 |
+
# -e SOC_TYPE=${SOC_TYPE} \
|
| 87 |
+
# -e BUILD_TYPE=${BUILD_TYPE} \
|
| 88 |
+
# -e USE_ACL_GRAPH=${USE_ACL_GRAPH} \
|
| 89 |
+
# "${{ steps.cann-image.outputs.image }}" \
|
| 90 |
+
# bash -lc '
|
| 91 |
+
# set -e
|
| 92 |
+
# yum install -y --setopt=install_weak_deps=False --setopt=tsflags=nodocs git gcc gcc-c++ make cmake openssl-devel
|
| 93 |
+
# yum clean all && rm -rf /var/cache/yum
|
| 94 |
+
# git config --global --add safe.directory "/workspace"
|
| 95 |
+
# export LD_LIBRARY_PATH=${ASCEND_TOOLKIT_HOME}/lib64:${ASCEND_TOOLKIT_HOME}/$(uname -m)-linux/devlib/:${LD_LIBRARY_PATH}
|
| 96 |
+
# cmake -S . -B build \
|
| 97 |
+
# -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
|
| 98 |
+
# -DGGML_CANN=on \
|
| 99 |
+
# -DSOC_TYPE=${SOC_TYPE} \
|
| 100 |
+
# -DUSE_ACL_GRAPH=${USE_ACL_GRAPH}
|
| 101 |
+
# cmake --build build -j $(nproc)
|
| 102 |
+
#
|
| 103 |
+
# chown -R '"${HOST_UID}"':'"${HOST_GID}"' /workspace/build
|
| 104 |
+
# '
|
.github/workflows/build-cmake-pkg.yml
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: Build relocatable cmake package
|
| 2 |
+
on:
|
| 3 |
+
workflow_dispatch:
|
| 4 |
+
workflow_call:
|
| 5 |
+
|
| 6 |
+
jobs:
|
| 7 |
+
linux:
|
| 8 |
+
runs-on: [self-hosted, Linux]
|
| 9 |
+
steps:
|
| 10 |
+
- uses: actions/checkout@v6
|
| 11 |
+
with:
|
| 12 |
+
fetch-depth: 0
|
| 13 |
+
|
| 14 |
+
- name: Build
|
| 15 |
+
run: |
|
| 16 |
+
PREFIX="$(pwd)"/inst
|
| 17 |
+
cmake -S . -B build \
|
| 18 |
+
-DCMAKE_PREFIX_PATH="$PREFIX" \
|
| 19 |
+
-DLLAMA_OPENSSL=OFF \
|
| 20 |
+
-DLLAMA_BUILD_TESTS=OFF \
|
| 21 |
+
-DLLAMA_BUILD_TOOLS=OFF \
|
| 22 |
+
-DLLAMA_BUILD_EXAMPLES=OFF \
|
| 23 |
+
-DLLAMA_BUILD_APP=OFF \
|
| 24 |
+
-DLLAMA_BUILD_IS_DEV=OFF \
|
| 25 |
+
-DCMAKE_BUILD_TYPE=Release
|
| 26 |
+
cmake --build build --config Release -j $(nproc)
|
| 27 |
+
cmake --install build --prefix "$PREFIX" --config Release
|
| 28 |
+
|
| 29 |
+
export LLAMA_CONFIG="$PREFIX"/lib/cmake/llama/llama-config.cmake
|
| 30 |
+
build_commit=$(git rev-parse --short HEAD | xargs)
|
| 31 |
+
build_number=$(git rev-list --count HEAD | xargs)
|
| 32 |
+
|
| 33 |
+
major=$(grep -oE "set\(LLAMA_VERSION_MAJOR[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
|
| 34 |
+
minor=$(grep -oE "set\(LLAMA_VERSION_MINOR[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
|
| 35 |
+
patch=$(grep -oE "set\(LLAMA_VERSION_PATCH[[:space:]]+[0-9]+" CMakeLists.txt | grep -oE "[0-9]+$")
|
| 36 |
+
build_version="$major.$minor.$patch"
|
| 37 |
+
|
| 38 |
+
checks=("set\(LLAMA_VERSION[[:space:]]+$build_version\)"
|
| 39 |
+
"set\(LLAMA_BUILD_COMMIT[[:space:]]+$build_commit\)"
|
| 40 |
+
"set\(LLAMA_BUILD_NUMBER[[:space:]]+$build_number\)")
|
| 41 |
+
|
| 42 |
+
for check in "${checks[@]}"; do
|
| 43 |
+
if ! grep -qE "$check" "$LLAMA_CONFIG"; then
|
| 44 |
+
echo "Checking llama-config.cmake version... \"$check\" failed!"
|
| 45 |
+
exit 1
|
| 46 |
+
fi
|
| 47 |
+
done
|
| 48 |
+
|
| 49 |
+
echo "Checking llama-config.cmake version... success."
|
| 50 |
+
|
| 51 |
+
cd examples/simple-cmake-pkg
|
| 52 |
+
cmake -S . -B build -DCMAKE_PREFIX_PATH="$PREFIX"/lib/cmake
|
| 53 |
+
cmake --build build -j $(nproc)
|
.github/workflows/build-cpu.yml
ADDED
|
@@ -0,0 +1,235 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: CI (cpu)
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
workflow_dispatch: # allows manual triggering
|
| 5 |
+
push:
|
| 6 |
+
branches:
|
| 7 |
+
- master
|
| 8 |
+
paths: [
|
| 9 |
+
'.github/workflows/build-cpu.yml',
|
| 10 |
+
'.github/workflows/build-cmake-pkg.yml',
|
| 11 |
+
'**/CMakeLists.txt',
|
| 12 |
+
'**/.cmake',
|
| 13 |
+
'**/*.h',
|
| 14 |
+
'**/*.hpp',
|
| 15 |
+
'**/*.c',
|
| 16 |
+
'**/*.cpp',
|
| 17 |
+
]
|
| 18 |
+
|
| 19 |
+
pull_request:
|
| 20 |
+
types: [opened, synchronize, reopened]
|
| 21 |
+
paths: [
|
| 22 |
+
'.github/workflows/build-cpu.yml',
|
| 23 |
+
'.github/workflows/build-cmake-pkg.yml',
|
| 24 |
+
'ggml/src/ggml-rpc/**',
|
| 25 |
+
'**/CMakeLists.txt',
|
| 26 |
+
'**/.cmake',
|
| 27 |
+
'**/*.h',
|
| 28 |
+
'**/*.hpp',
|
| 29 |
+
'**/*.c',
|
| 30 |
+
'**/*.cpp'
|
| 31 |
+
]
|
| 32 |
+
|
| 33 |
+
concurrency:
|
| 34 |
+
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
| 35 |
+
cancel-in-progress: true
|
| 36 |
+
|
| 37 |
+
env:
|
| 38 |
+
GGML_NLOOP: 3
|
| 39 |
+
GGML_N_THREADS: 1
|
| 40 |
+
LLAMA_ARG_LOG_COLORS: 1
|
| 41 |
+
LLAMA_ARG_LOG_PREFIX: 1
|
| 42 |
+
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
| 43 |
+
|
| 44 |
+
jobs:
|
| 45 |
+
build-cmake-pkg:
|
| 46 |
+
uses: ./.github/workflows/build-cmake-pkg.yml
|
| 47 |
+
|
| 48 |
+
ubuntu:
|
| 49 |
+
strategy:
|
| 50 |
+
matrix:
|
| 51 |
+
include:
|
| 52 |
+
- build: 'x64'
|
| 53 |
+
os: ubuntu-22.04
|
| 54 |
+
- build: 'arm64'
|
| 55 |
+
os: ubuntu-24.04-arm
|
| 56 |
+
|
| 57 |
+
runs-on: ${{ matrix.os }}
|
| 58 |
+
|
| 59 |
+
steps:
|
| 60 |
+
- name: Clone
|
| 61 |
+
id: checkout
|
| 62 |
+
uses: actions/checkout@v6
|
| 63 |
+
|
| 64 |
+
- name: ccache
|
| 65 |
+
uses: ggml-org/ccache-action@v1.2.24
|
| 66 |
+
with:
|
| 67 |
+
key: cpu-${{ matrix.os }}
|
| 68 |
+
save: false
|
| 69 |
+
|
| 70 |
+
- name: Build Dependencies
|
| 71 |
+
id: build_depends
|
| 72 |
+
run: |
|
| 73 |
+
sudo apt-get update
|
| 74 |
+
sudo apt-get install -y --no-install-recommends \
|
| 75 |
+
python3 python3-pip python3-dev python3-wheel \
|
| 76 |
+
libjpeg-dev build-essential libssl-dev \
|
| 77 |
+
git-lfs
|
| 78 |
+
|
| 79 |
+
- name: Toolchain workaround (GCC 14)
|
| 80 |
+
if: ${{ contains(matrix.os, 'ubuntu-24.04') }}
|
| 81 |
+
run: |
|
| 82 |
+
sudo apt-get install -y gcc-14 g++-14
|
| 83 |
+
echo "CC=gcc-14" >> "$GITHUB_ENV"
|
| 84 |
+
echo "CXX=g++-14" >> "$GITHUB_ENV"
|
| 85 |
+
|
| 86 |
+
- name: Python Dependencies
|
| 87 |
+
id: python_depends
|
| 88 |
+
run: |
|
| 89 |
+
export PIP_BREAK_SYSTEM_PACKAGES="1"
|
| 90 |
+
python3 -m pip install --upgrade pip setuptools
|
| 91 |
+
pip3 install ./gguf-py
|
| 92 |
+
|
| 93 |
+
- name: ccache-buckets-restore
|
| 94 |
+
uses: ./.github/actions/ccache-buckets
|
| 95 |
+
env:
|
| 96 |
+
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
| 97 |
+
with:
|
| 98 |
+
key: cpu-${{ matrix.os }}
|
| 99 |
+
folder: llama.cpp
|
| 100 |
+
hf_bucket: ggml-org/cache
|
| 101 |
+
|
| 102 |
+
- name: Build
|
| 103 |
+
id: cmake_build
|
| 104 |
+
run: |
|
| 105 |
+
cmake -B build \
|
| 106 |
+
-DGGML_NATIVE=OFF \
|
| 107 |
+
-DLLAMA_FATAL_WARNINGS=ON \
|
| 108 |
+
-DGGML_RPC=ON
|
| 109 |
+
time cmake --build build --config Release -j $(nproc)
|
| 110 |
+
|
| 111 |
+
- name: ccache-buckets-save
|
| 112 |
+
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
| 113 |
+
uses: ./.github/actions/ccache-buckets
|
| 114 |
+
env:
|
| 115 |
+
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
| 116 |
+
with:
|
| 117 |
+
key: cpu-${{ matrix.os }}
|
| 118 |
+
folder: llama.cpp
|
| 119 |
+
evict-old-files: 1d
|
| 120 |
+
hf_bucket: ggml-org/cache
|
| 121 |
+
save: true
|
| 122 |
+
|
| 123 |
+
- name: Test
|
| 124 |
+
id: cmake_test
|
| 125 |
+
run: |
|
| 126 |
+
cd build
|
| 127 |
+
ctest -L main --verbose --timeout 900
|
| 128 |
+
|
| 129 |
+
- name: Test llama2c conversion
|
| 130 |
+
id: llama2c_test
|
| 131 |
+
run: |
|
| 132 |
+
cd build
|
| 133 |
+
echo "Fetch tokenizer"
|
| 134 |
+
wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/tok512.bin
|
| 135 |
+
echo "Fetch llama2c model"
|
| 136 |
+
wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/stories260K.bin
|
| 137 |
+
./bin/llama-convert-llama2c-to-ggml --copy-vocab-from-model ./tok512.bin --llama2c-model stories260K.bin --llama2c-output-model stories260K.gguf
|
| 138 |
+
./bin/llama-completion -m stories260K.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256
|
| 139 |
+
|
| 140 |
+
windows:
|
| 141 |
+
name: windows / ${{ matrix.build }}
|
| 142 |
+
runs-on: windows-2025
|
| 143 |
+
|
| 144 |
+
env:
|
| 145 |
+
OPENBLAS_VERSION: 0.3.23
|
| 146 |
+
SDE_VERSION: 9.33.0-2024-01-07
|
| 147 |
+
|
| 148 |
+
strategy:
|
| 149 |
+
matrix:
|
| 150 |
+
include:
|
| 151 |
+
- build: 'x64-cpu-static'
|
| 152 |
+
arch: 'x64'
|
| 153 |
+
defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DGGML_NATIVE=OFF -DGGML_OPENMP_FETCH=ON -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=OFF'
|
| 154 |
+
- build: 'x64-openblas'
|
| 155 |
+
arch: 'x64'
|
| 156 |
+
defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_BACKEND_DL=ON -DGGML_CPU_ALL_VARIANTS=ON -DGGML_OPENMP=OFF -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/openblas.lib"'
|
| 157 |
+
- build: 'arm64'
|
| 158 |
+
arch: 'arm64'
|
| 159 |
+
defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DGGML_NATIVE=OFF -DGGML_OPENMP_FETCH=ON -DLLAMA_BUILD_SERVER=ON'
|
| 160 |
+
|
| 161 |
+
steps:
|
| 162 |
+
- name: Clone
|
| 163 |
+
id: checkout
|
| 164 |
+
uses: actions/checkout@v6
|
| 165 |
+
|
| 166 |
+
- name: ccache
|
| 167 |
+
uses: ggml-org/ccache-action@v1.2.24
|
| 168 |
+
with:
|
| 169 |
+
key: cpu-windows-2025-${{ matrix.build }}
|
| 170 |
+
variant: ccache
|
| 171 |
+
evict-old-files: 1d
|
| 172 |
+
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
| 173 |
+
|
| 174 |
+
- name: Download OpenBLAS
|
| 175 |
+
id: get_openblas
|
| 176 |
+
if: ${{ matrix.build == 'x64-openblas' }}
|
| 177 |
+
run: |
|
| 178 |
+
curl.exe -o $env:RUNNER_TEMP/openblas.zip -L "https://github.com/xianyi/OpenBLAS/releases/download/v${env:OPENBLAS_VERSION}/OpenBLAS-${env:OPENBLAS_VERSION}-x64.zip"
|
| 179 |
+
curl.exe -o $env:RUNNER_TEMP/OpenBLAS.LICENSE.txt -L "https://github.com/xianyi/OpenBLAS/raw/v${env:OPENBLAS_VERSION}/LICENSE"
|
| 180 |
+
mkdir $env:RUNNER_TEMP/openblas
|
| 181 |
+
tar.exe -xvf $env:RUNNER_TEMP/openblas.zip -C $env:RUNNER_TEMP/openblas
|
| 182 |
+
$vcdir = $(vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath)
|
| 183 |
+
$msvc = $(join-path $vcdir $('VC\Tools\MSVC\'+$(gc -raw $(join-path $vcdir 'VC\Auxiliary\Build\Microsoft.VCToolsVersion.default.txt')).Trim()))
|
| 184 |
+
$lib = $(join-path $msvc 'bin\Hostx64\x64\lib.exe')
|
| 185 |
+
& $lib /machine:x64 "/def:${env:RUNNER_TEMP}/openblas/lib/libopenblas.def" "/out:${env:RUNNER_TEMP}/openblas/lib/openblas.lib" /name:openblas.dll
|
| 186 |
+
|
| 187 |
+
- name: Install Ninja
|
| 188 |
+
id: install_ninja
|
| 189 |
+
run: |
|
| 190 |
+
choco install ninja
|
| 191 |
+
|
| 192 |
+
- name: Build
|
| 193 |
+
id: cmake_build
|
| 194 |
+
run: |
|
| 195 |
+
cmake -S . -B build ${{ matrix.defines }} `
|
| 196 |
+
-DLLAMA_BUILD_BORINGSSL=ON
|
| 197 |
+
cmake --build build --config Release -j ${env:NUMBER_OF_PROCESSORS}
|
| 198 |
+
|
| 199 |
+
- name: Add libopenblas.dll
|
| 200 |
+
id: add_libopenblas_dll
|
| 201 |
+
if: ${{ matrix.build == 'x64-openblas' }}
|
| 202 |
+
run: |
|
| 203 |
+
cp $env:RUNNER_TEMP/openblas/bin/libopenblas.dll ./build/bin/Release/openblas.dll
|
| 204 |
+
cp $env:RUNNER_TEMP/OpenBLAS.LICENSE.txt ./build/bin/Release/OpenBLAS-${env:OPENBLAS_VERSION}.txt
|
| 205 |
+
|
| 206 |
+
- name: Test
|
| 207 |
+
id: cmake_test
|
| 208 |
+
if: ${{ matrix.arch == 'x64' }}
|
| 209 |
+
run: |
|
| 210 |
+
cd build
|
| 211 |
+
ctest -L main -C Release --verbose --timeout 900
|
| 212 |
+
|
| 213 |
+
# TODO: disabled for now, consider adding tests for all CPU variants instead
|
| 214 |
+
# - name: Test (Intel SDE)
|
| 215 |
+
# id: cmake_test_sde
|
| 216 |
+
# if: ${{ matrix.build == 'avx512-x64' && env.HAS_AVX512F == '0' }} # use Intel SDE for AVX-512 emulation
|
| 217 |
+
# run: |
|
| 218 |
+
# curl.exe -o $env:RUNNER_TEMP/sde.tar.xz -L "https://downloadmirror.intel.com/813591/sde-external-${env:SDE_VERSION}-win.tar.xz"
|
| 219 |
+
# # for some weird reason windows tar doesn't like sde tar.xz
|
| 220 |
+
# 7z x "-o${env:RUNNER_TEMP}" $env:RUNNER_TEMP/sde.tar.xz
|
| 221 |
+
# 7z x "-o${env:RUNNER_TEMP}" $env:RUNNER_TEMP/sde.tar
|
| 222 |
+
# $sde = $(join-path $env:RUNNER_TEMP sde-external-${env:SDE_VERSION}-win/sde.exe)
|
| 223 |
+
# cd build
|
| 224 |
+
# $env:LLAMA_SKIP_TESTS_SLOW_ON_EMULATOR = 1
|
| 225 |
+
# & $sde -future -- ctest -L main -C Release --verbose --timeout 900
|
| 226 |
+
|
| 227 |
+
- name: ccache-clear
|
| 228 |
+
uses: ./.github/actions/ccache-clear
|
| 229 |
+
env:
|
| 230 |
+
GH_TOKEN: ${{ github.token }}
|
| 231 |
+
with:
|
| 232 |
+
key: cpu-windows-2025-${{ matrix.build }}
|
| 233 |
+
older: 5m
|
| 234 |
+
min: 1
|
| 235 |
+
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
|
.github/workflows/build-cross.yml
ADDED
|
@@ -0,0 +1,317 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: CI (cross)
|
| 2 |
+
on:
|
| 3 |
+
# only manual triggers due to low-importance of the workflows
|
| 4 |
+
# TODO: for regular runs, provision dedicated self-hosted runners
|
| 5 |
+
workflow_dispatch:
|
| 6 |
+
push:
|
| 7 |
+
branches:
|
| 8 |
+
- master
|
| 9 |
+
paths: [
|
| 10 |
+
'.github/workflows/build-cross.yml',
|
| 11 |
+
'ggml/src/spacemit/*',
|
| 12 |
+
'ggml/src/arch/loongarch/*'
|
| 13 |
+
]
|
| 14 |
+
# run once every week
|
| 15 |
+
schedule:
|
| 16 |
+
- cron: '0 0 * * 0'
|
| 17 |
+
|
| 18 |
+
concurrency:
|
| 19 |
+
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
| 20 |
+
cancel-in-progress: true
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
jobs:
|
| 24 |
+
# ubuntu-24-riscv64-cpu-cross:
|
| 25 |
+
# runs-on: ubuntu-24.04
|
| 26 |
+
|
| 27 |
+
# steps:
|
| 28 |
+
# - uses: actions/checkout@v6
|
| 29 |
+
# - name: Setup Riscv
|
| 30 |
+
# run: |
|
| 31 |
+
# sudo dpkg --add-architecture riscv64
|
| 32 |
+
|
| 33 |
+
# # Add arch-specific repositories for non-amd64 architectures
|
| 34 |
+
# cat << EOF | sudo tee /etc/apt/sources.list.d/riscv64-ports.list
|
| 35 |
+
# deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ noble main universe
|
| 36 |
+
# deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ noble-updates main universe
|
| 37 |
+
# deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ noble-security main universe
|
| 38 |
+
# deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ noble-backports main universe
|
| 39 |
+
# EOF
|
| 40 |
+
|
| 41 |
+
# sudo apt-get update || true ;# Prevent failure due to missing URLs.
|
| 42 |
+
|
| 43 |
+
# sudo apt-get install -y --no-install-recommends \
|
| 44 |
+
# build-essential \
|
| 45 |
+
# gcc-14-riscv64-linux-gnu \
|
| 46 |
+
# g++-14-riscv64-linux-gnu
|
| 47 |
+
|
| 48 |
+
# - name: Build
|
| 49 |
+
# run: |
|
| 50 |
+
# cmake -B build -DLLAMA_OPENSSL=OFF \
|
| 51 |
+
# -DCMAKE_BUILD_TYPE=Release \
|
| 52 |
+
# -DGGML_OPENMP=OFF \
|
| 53 |
+
# -DLLAMA_BUILD_EXAMPLES=ON \
|
| 54 |
+
# -DLLAMA_BUILD_TOOLS=ON \
|
| 55 |
+
# -DLLAMA_BUILD_TESTS=OFF \
|
| 56 |
+
# -DCMAKE_SYSTEM_NAME=Linux \
|
| 57 |
+
# -DCMAKE_SYSTEM_PROCESSOR=riscv64 \
|
| 58 |
+
# -DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \
|
| 59 |
+
# -DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14 \
|
| 60 |
+
# -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
|
| 61 |
+
# -DCMAKE_FIND_ROOT_PATH=/usr/lib/riscv64-linux-gnu \
|
| 62 |
+
# -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \
|
| 63 |
+
# -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY \
|
| 64 |
+
# -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH
|
| 65 |
+
|
| 66 |
+
# cmake --build build --config Release -j $(nproc)
|
| 67 |
+
|
| 68 |
+
# ubuntu-24-riscv64-vulkan-cross:
|
| 69 |
+
# runs-on: ubuntu-24.04
|
| 70 |
+
|
| 71 |
+
# steps:
|
| 72 |
+
# - uses: actions/checkout@v6
|
| 73 |
+
# - name: Setup Riscv
|
| 74 |
+
# run: |
|
| 75 |
+
# sudo dpkg --add-architecture riscv64
|
| 76 |
+
|
| 77 |
+
# # Add arch-specific repositories for non-amd64 architectures
|
| 78 |
+
# cat << EOF | sudo tee /etc/apt/sources.list.d/riscv64-ports.list
|
| 79 |
+
# deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ noble main universe
|
| 80 |
+
# deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ noble-updates main universe
|
| 81 |
+
# deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ noble-security main universe
|
| 82 |
+
# deb [arch=riscv64] http://ports.ubuntu.com/ubuntu-ports/ noble-backports main universe
|
| 83 |
+
# EOF
|
| 84 |
+
|
| 85 |
+
# sudo apt-get update || true ;# Prevent failure due to missing URLs.
|
| 86 |
+
|
| 87 |
+
# sudo apt-get install -y --no-install-recommends \
|
| 88 |
+
# build-essential \
|
| 89 |
+
# glslc \
|
| 90 |
+
# gcc-14-riscv64-linux-gnu \
|
| 91 |
+
# g++-14-riscv64-linux-gnu \
|
| 92 |
+
# libvulkan-dev:riscv64
|
| 93 |
+
|
| 94 |
+
# - name: Build
|
| 95 |
+
# run: |
|
| 96 |
+
# cmake -B build -DLLAMA_OPENSSL=OFF \
|
| 97 |
+
# -DCMAKE_BUILD_TYPE=Release \
|
| 98 |
+
# -DGGML_VULKAN=ON \
|
| 99 |
+
# -DGGML_OPENMP=OFF \
|
| 100 |
+
# -DLLAMA_BUILD_EXAMPLES=ON \
|
| 101 |
+
# -DLLAMA_BUILD_TOOLS=ON \
|
| 102 |
+
# -DLLAMA_BUILD_TESTS=OFF \
|
| 103 |
+
# -DCMAKE_SYSTEM_NAME=Linux \
|
| 104 |
+
# -DCMAKE_SYSTEM_PROCESSOR=riscv64 \
|
| 105 |
+
# -DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \
|
| 106 |
+
# -DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14 \
|
| 107 |
+
# -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
|
| 108 |
+
# -DCMAKE_FIND_ROOT_PATH=/usr/lib/riscv64-linux-gnu \
|
| 109 |
+
# -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \
|
| 110 |
+
# -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY \
|
| 111 |
+
# -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH
|
| 112 |
+
|
| 113 |
+
# cmake --build build --config Release -j $(nproc)
|
| 114 |
+
|
| 115 |
+
# ubuntu-24-arm64-vulkan-cross:
|
| 116 |
+
# runs-on: ubuntu-24.04
|
| 117 |
+
|
| 118 |
+
# steps:
|
| 119 |
+
# - uses: actions/checkout@v6
|
| 120 |
+
# - name: Setup Arm64
|
| 121 |
+
# run: |
|
| 122 |
+
# sudo dpkg --add-architecture arm64
|
| 123 |
+
|
| 124 |
+
# # Add arch-specific repositories for non-amd64 architectures
|
| 125 |
+
# cat << EOF | sudo tee /etc/apt/sources.list.d/arm64-ports.list
|
| 126 |
+
# deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ noble main universe
|
| 127 |
+
# deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ noble-updates main universe
|
| 128 |
+
# deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ noble-security main universe
|
| 129 |
+
# deb [arch=arm64] http://ports.ubuntu.com/ubuntu-ports/ noble-backports main universe
|
| 130 |
+
# EOF
|
| 131 |
+
|
| 132 |
+
# sudo apt-get update || true ;# Prevent failure due to missing URLs.
|
| 133 |
+
|
| 134 |
+
# sudo apt-get install -y --no-install-recommends \
|
| 135 |
+
# build-essential \
|
| 136 |
+
# glslc \
|
| 137 |
+
# crossbuild-essential-arm64 \
|
| 138 |
+
# libvulkan-dev:arm64
|
| 139 |
+
|
| 140 |
+
# - name: Build
|
| 141 |
+
# run: |
|
| 142 |
+
# cmake -B build -DLLAMA_OPENSSL=OFF \
|
| 143 |
+
# -DCMAKE_BUILD_TYPE=Release \
|
| 144 |
+
# -DGGML_VULKAN=ON \
|
| 145 |
+
# -DGGML_OPENMP=OFF \
|
| 146 |
+
# -DLLAMA_BUILD_EXAMPLES=ON \
|
| 147 |
+
# -DLLAMA_BUILD_TOOLS=ON \
|
| 148 |
+
# -DLLAMA_BUILD_TESTS=OFF \
|
| 149 |
+
# -DCMAKE_SYSTEM_NAME=Linux \
|
| 150 |
+
# -DCMAKE_SYSTEM_PROCESSOR=aarch64 \
|
| 151 |
+
# -DCMAKE_C_COMPILER=aarch64-linux-gnu-gcc \
|
| 152 |
+
# -DCMAKE_CXX_COMPILER=aarch64-linux-gnu-g++ \
|
| 153 |
+
# -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
|
| 154 |
+
# -DCMAKE_FIND_ROOT_PATH=/usr/lib/aarch64-linux-gnu \
|
| 155 |
+
# -DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \
|
| 156 |
+
# -DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY \
|
| 157 |
+
# -DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH
|
| 158 |
+
|
| 159 |
+
# cmake --build build --config Release -j $(nproc)
|
| 160 |
+
|
| 161 |
+
debian-13-loongarch64-cpu-cross:
|
| 162 |
+
runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
|
| 163 |
+
container: debian@sha256:653dfb9f86c3782e8369d5f7d29bb8faba1f4bff9025db46e807fa4c22903671
|
| 164 |
+
|
| 165 |
+
steps:
|
| 166 |
+
- uses: actions/checkout@v6
|
| 167 |
+
- name: Setup LoongArch
|
| 168 |
+
run: |
|
| 169 |
+
rm -f /etc/apt/sources.list.d/*
|
| 170 |
+
cat << EOF | tee /etc/apt/sources.list.d/debian-ports.list
|
| 171 |
+
deb http://snapshot.debian.org/archive/debian/20250515T202920Z/ trixie main
|
| 172 |
+
EOF
|
| 173 |
+
( echo 'quiet "true";'; \
|
| 174 |
+
echo 'APT::Get::Assume-Yes "true";'; \
|
| 175 |
+
echo 'APT::Install-Recommends "false";'; \
|
| 176 |
+
echo 'Acquire::Check-Valid-Until "false";'; \
|
| 177 |
+
echo 'Acquire::Retries "5";'; \
|
| 178 |
+
) > /etc/apt/apt.conf.d/99snapshot-repos
|
| 179 |
+
|
| 180 |
+
apt-get update
|
| 181 |
+
apt-get install -y ca-certificates debian-ports-archive-keyring cmake git zip
|
| 182 |
+
dpkg --add-architecture loong64
|
| 183 |
+
|
| 184 |
+
# Add arch-specific repositories for non-amd64 architectures
|
| 185 |
+
cat << EOF | tee /etc/apt/sources.list.d/loong64-ports.list
|
| 186 |
+
deb [arch=loong64] http://snapshot.debian.org/archive/debian-ports/20250515T194251Z/ sid main
|
| 187 |
+
EOF
|
| 188 |
+
|
| 189 |
+
apt-get update || true ;# Prevent failure due to missing URLs.
|
| 190 |
+
|
| 191 |
+
apt-get install -y --no-install-recommends \
|
| 192 |
+
build-essential \
|
| 193 |
+
gcc-14-loongarch64-linux-gnu \
|
| 194 |
+
g++-14-loongarch64-linux-gnu
|
| 195 |
+
|
| 196 |
+
- name: Build
|
| 197 |
+
run: |
|
| 198 |
+
cmake -B build -DLLAMA_OPENSSL=OFF \
|
| 199 |
+
-DCMAKE_BUILD_TYPE=Release \
|
| 200 |
+
-DGGML_OPENMP=OFF \
|
| 201 |
+
-DLLAMA_BUILD_EXAMPLES=ON \
|
| 202 |
+
-DLLAMA_BUILD_TOOLS=ON \
|
| 203 |
+
-DLLAMA_BUILD_TESTS=OFF \
|
| 204 |
+
-DCMAKE_SYSTEM_NAME=Linux \
|
| 205 |
+
-DCMAKE_SYSTEM_PROCESSOR=loongarch64 \
|
| 206 |
+
-DCMAKE_C_COMPILER=loongarch64-linux-gnu-gcc-14 \
|
| 207 |
+
-DCMAKE_CXX_COMPILER=loongarch64-linux-gnu-g++-14 \
|
| 208 |
+
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
|
| 209 |
+
-DCMAKE_FIND_ROOT_PATH=/usr/lib/loongarch64-linux-gnu \
|
| 210 |
+
-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \
|
| 211 |
+
-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY \
|
| 212 |
+
-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH
|
| 213 |
+
|
| 214 |
+
cmake --build build --config Release -j $(nproc)
|
| 215 |
+
|
| 216 |
+
debian-13-loongarch64-vulkan-cross:
|
| 217 |
+
runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
|
| 218 |
+
container: debian@sha256:653dfb9f86c3782e8369d5f7d29bb8faba1f4bff9025db46e807fa4c22903671
|
| 219 |
+
|
| 220 |
+
steps:
|
| 221 |
+
- uses: actions/checkout@v6
|
| 222 |
+
- name: Setup LoongArch
|
| 223 |
+
run: |
|
| 224 |
+
rm -f /etc/apt/sources.list.d/*
|
| 225 |
+
cat << EOF | tee /etc/apt/sources.list.d/debian-ports.list
|
| 226 |
+
deb http://snapshot.debian.org/archive/debian/20250515T202920Z/ trixie main
|
| 227 |
+
EOF
|
| 228 |
+
( echo 'quiet "true";'; \
|
| 229 |
+
echo 'APT::Get::Assume-Yes "true";'; \
|
| 230 |
+
echo 'APT::Install-Recommends "false";'; \
|
| 231 |
+
echo 'Acquire::Check-Valid-Until "false";'; \
|
| 232 |
+
echo 'Acquire::Retries "5";'; \
|
| 233 |
+
) > /etc/apt/apt.conf.d/99snapshot-repos
|
| 234 |
+
|
| 235 |
+
apt-get update
|
| 236 |
+
apt-get install -y ca-certificates debian-ports-archive-keyring cmake git zip
|
| 237 |
+
dpkg --add-architecture loong64
|
| 238 |
+
|
| 239 |
+
# Add arch-specific repositories for non-amd64 architectures
|
| 240 |
+
cat << EOF | tee /etc/apt/sources.list.d/loong64-ports.list
|
| 241 |
+
deb [arch=loong64] http://snapshot.debian.org/archive/debian-ports/20250515T194251Z/ sid main
|
| 242 |
+
EOF
|
| 243 |
+
|
| 244 |
+
apt-get update || true ;# Prevent failure due to missing URLs.
|
| 245 |
+
|
| 246 |
+
apt-get install -y --no-install-recommends \
|
| 247 |
+
build-essential \
|
| 248 |
+
glslc \
|
| 249 |
+
spirv-headers \
|
| 250 |
+
gcc-14-loongarch64-linux-gnu \
|
| 251 |
+
g++-14-loongarch64-linux-gnu \
|
| 252 |
+
libvulkan-dev:loong64
|
| 253 |
+
|
| 254 |
+
- name: Build
|
| 255 |
+
run: |
|
| 256 |
+
cmake -B build -DLLAMA_OPENSSL=OFF \
|
| 257 |
+
-DCMAKE_BUILD_TYPE=Release \
|
| 258 |
+
-DGGML_VULKAN=ON \
|
| 259 |
+
-DGGML_OPENMP=OFF \
|
| 260 |
+
-DLLAMA_BUILD_EXAMPLES=ON \
|
| 261 |
+
-DLLAMA_BUILD_TOOLS=ON \
|
| 262 |
+
-DLLAMA_BUILD_TESTS=OFF \
|
| 263 |
+
-DCMAKE_SYSTEM_NAME=Linux \
|
| 264 |
+
-DCMAKE_SYSTEM_PROCESSOR=loongarch64 \
|
| 265 |
+
-DCMAKE_C_COMPILER=loongarch64-linux-gnu-gcc-14 \
|
| 266 |
+
-DCMAKE_CXX_COMPILER=loongarch64-linux-gnu-g++-14 \
|
| 267 |
+
-DCMAKE_POSITION_INDEPENDENT_CODE=ON \
|
| 268 |
+
-DCMAKE_FIND_ROOT_PATH=/usr/lib/loongarch64-linux-gnu \
|
| 269 |
+
-DCMAKE_FIND_ROOT_PATH_MODE_PROGRAM=NEVER \
|
| 270 |
+
-DCMAKE_FIND_ROOT_PATH_MODE_LIBRARY=ONLY \
|
| 271 |
+
-DCMAKE_FIND_ROOT_PATH_MODE_INCLUDE=BOTH
|
| 272 |
+
|
| 273 |
+
cmake --build build --config Release -j $(nproc)
|
| 274 |
+
|
| 275 |
+
ubuntu-24-riscv64-cpu-spacemit-ime-cross:
|
| 276 |
+
runs-on: ubuntu-24.04
|
| 277 |
+
|
| 278 |
+
env:
|
| 279 |
+
# Make sure this is in sync with build-cache.yml
|
| 280 |
+
SPACEMIT_IME_TOOLCHAIN_VERSION: "1.2.4"
|
| 281 |
+
|
| 282 |
+
steps:
|
| 283 |
+
- uses: actions/checkout@v6
|
| 284 |
+
|
| 285 |
+
#- name: Use SpacemiT Toolchain Cache
|
| 286 |
+
# uses: actions/cache@v5
|
| 287 |
+
# id: cache-toolchain
|
| 288 |
+
# with:
|
| 289 |
+
# path: ./spacemit_toolchain
|
| 290 |
+
# key: cache-gha-spacemit-ime-toolchain-v${{ env.SPACEMIT_IME_TOOLCHAIN_VERSION }}-${{ runner.os }}
|
| 291 |
+
|
| 292 |
+
- name: Setup SpacemiT Toolchain
|
| 293 |
+
#if: steps.cache-toolchain.outputs.cache-hit != 'true'
|
| 294 |
+
uses: ./.github/actions/linux-setup-spacemit
|
| 295 |
+
with:
|
| 296 |
+
path: ./spacemit_toolchain
|
| 297 |
+
version: ${{ env.SPACEMIT_IME_TOOLCHAIN_VERSION }}
|
| 298 |
+
|
| 299 |
+
- name: Build
|
| 300 |
+
run: |
|
| 301 |
+
export RISCV_ROOT_PATH=${PWD}/spacemit_toolchain
|
| 302 |
+
cmake -B build -DLLAMA_OPENSSL=OFF \
|
| 303 |
+
-DCMAKE_BUILD_TYPE=Release \
|
| 304 |
+
-DLLAMA_BUILD_EXAMPLES=ON \
|
| 305 |
+
-DGGML_CPU_REPACK=OFF \
|
| 306 |
+
-DLLAMA_BUILD_TOOLS=ON \
|
| 307 |
+
-DLLAMA_BUILD_TESTS=OFF \
|
| 308 |
+
-DGGML_CPU_RISCV64_SPACEMIT=ON \
|
| 309 |
+
-DGGML_RVV=ON \
|
| 310 |
+
-DGGML_RV_ZVFH=ON \
|
| 311 |
+
-DGGML_RV_ZFH=ON \
|
| 312 |
+
-DGGML_RV_ZICBOP=ON \
|
| 313 |
+
-DGGML_RV_ZIHINTPAUSE=ON \
|
| 314 |
+
-DGGML_RV_ZBA=ON \
|
| 315 |
+
-DCMAKE_TOOLCHAIN_FILE=${PWD}/cmake/riscv64-spacemit-linux-gnu-gcc.cmake
|
| 316 |
+
|
| 317 |
+
cmake --build build --config Release -j $(nproc)
|
.github/workflows/build-cuda-ubuntu.yml
ADDED
|
@@ -0,0 +1,193 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: CI (CUDA, ubuntu)
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
workflow_dispatch: # allows manual triggering
|
| 5 |
+
push:
|
| 6 |
+
branches:
|
| 7 |
+
- master
|
| 8 |
+
paths: [
|
| 9 |
+
'.github/workflows/build-cuda-ubuntu.yml',
|
| 10 |
+
'**/CMakeLists.txt',
|
| 11 |
+
'**/.cmake',
|
| 12 |
+
'**/*.h',
|
| 13 |
+
'**/*.hpp',
|
| 14 |
+
'**/*.c',
|
| 15 |
+
'**/*.cpp',
|
| 16 |
+
'**/*.cu',
|
| 17 |
+
'**/*.cuh'
|
| 18 |
+
]
|
| 19 |
+
|
| 20 |
+
pull_request:
|
| 21 |
+
types: [opened, synchronize, reopened]
|
| 22 |
+
paths: [
|
| 23 |
+
'.github/workflows/build-cuda-ubuntu.yml',
|
| 24 |
+
'ggml/src/ggml-cuda/**'
|
| 25 |
+
]
|
| 26 |
+
|
| 27 |
+
concurrency:
|
| 28 |
+
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
| 29 |
+
cancel-in-progress: true
|
| 30 |
+
|
| 31 |
+
env:
|
| 32 |
+
GGML_NLOOP: 3
|
| 33 |
+
GGML_N_THREADS: 1
|
| 34 |
+
LLAMA_ARG_LOG_COLORS: 1
|
| 35 |
+
LLAMA_ARG_LOG_PREFIX: 1
|
| 36 |
+
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
| 37 |
+
|
| 38 |
+
jobs:
|
| 39 |
+
cuda:
|
| 40 |
+
runs-on: ubuntu-24.04
|
| 41 |
+
container: nvidia/cuda:12.6.2-devel-ubuntu24.04
|
| 42 |
+
|
| 43 |
+
steps:
|
| 44 |
+
- name: Clone
|
| 45 |
+
id: checkout
|
| 46 |
+
uses: actions/checkout@v6
|
| 47 |
+
|
| 48 |
+
- name: Install dependencies
|
| 49 |
+
env:
|
| 50 |
+
DEBIAN_FRONTEND: noninteractive
|
| 51 |
+
run: |
|
| 52 |
+
apt update
|
| 53 |
+
apt install -y cmake build-essential ninja-build libgomp1 git libssl-dev jq python3 python3-venv python3-pip
|
| 54 |
+
|
| 55 |
+
- name: ccache
|
| 56 |
+
uses: ggml-org/ccache-action@v1.2.24
|
| 57 |
+
with:
|
| 58 |
+
key: cuda-ubuntu-24.04-cuda
|
| 59 |
+
save: false
|
| 60 |
+
|
| 61 |
+
- name: ccache-buckets-restore
|
| 62 |
+
uses: ./.github/actions/ccache-buckets
|
| 63 |
+
env:
|
| 64 |
+
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
| 65 |
+
with:
|
| 66 |
+
key: cuda-ubuntu-24.04-cuda
|
| 67 |
+
folder: llama.cpp
|
| 68 |
+
hf_bucket: ggml-org/cache
|
| 69 |
+
|
| 70 |
+
- name: Build with CMake
|
| 71 |
+
# TODO: Remove GGML_CUDA_CUB_3DOT2 flag once CCCL 3.2 is bundled within CTK and that CTK version is used in this project
|
| 72 |
+
run: |
|
| 73 |
+
cmake -S . -B build -G Ninja \
|
| 74 |
+
-DLLAMA_FATAL_WARNINGS=ON \
|
| 75 |
+
-DCMAKE_BUILD_TYPE=Release \
|
| 76 |
+
-DCMAKE_CUDA_ARCHITECTURES=89-real \
|
| 77 |
+
-DCMAKE_EXE_LINKER_FLAGS=-Wl,--allow-shlib-undefined \
|
| 78 |
+
-DGGML_NATIVE=OFF \
|
| 79 |
+
-DGGML_CUDA=ON \
|
| 80 |
+
-DGGML_CUDA_CUB_3DOT2=ON
|
| 81 |
+
cmake --build build
|
| 82 |
+
|
| 83 |
+
- name: ccache-buckets-save
|
| 84 |
+
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
| 85 |
+
uses: ./.github/actions/ccache-buckets
|
| 86 |
+
env:
|
| 87 |
+
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
| 88 |
+
with:
|
| 89 |
+
key: cuda-ubuntu-24.04-cuda
|
| 90 |
+
folder: llama.cpp
|
| 91 |
+
evict-old-files: 1d
|
| 92 |
+
hf_bucket: ggml-org/cache
|
| 93 |
+
save: true
|
| 94 |
+
|
| 95 |
+
hip:
|
| 96 |
+
runs-on: ubuntu-22.04
|
| 97 |
+
container: rocm/dev-ubuntu-22.04:6.1.2
|
| 98 |
+
|
| 99 |
+
steps:
|
| 100 |
+
- name: Clone
|
| 101 |
+
id: checkout
|
| 102 |
+
uses: actions/checkout@v6
|
| 103 |
+
|
| 104 |
+
- name: Dependencies
|
| 105 |
+
id: depends
|
| 106 |
+
run: |
|
| 107 |
+
sudo apt-get update
|
| 108 |
+
sudo apt-get install -y build-essential git cmake rocblas-dev hipblas-dev libssl-dev rocwmma-dev jq python3-venv
|
| 109 |
+
|
| 110 |
+
- name: ccache
|
| 111 |
+
uses: ggml-org/ccache-action@v1.2.24
|
| 112 |
+
with:
|
| 113 |
+
key: cuda-ubuntu-22.04-hip
|
| 114 |
+
save: false
|
| 115 |
+
|
| 116 |
+
- name: ccache-buckets-restore
|
| 117 |
+
uses: ./.github/actions/ccache-buckets
|
| 118 |
+
env:
|
| 119 |
+
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
| 120 |
+
with:
|
| 121 |
+
key: cuda-ubuntu-22.04-hip
|
| 122 |
+
folder: llama.cpp
|
| 123 |
+
hf_bucket: ggml-org/cache
|
| 124 |
+
|
| 125 |
+
- name: Build with native CMake HIP support
|
| 126 |
+
id: cmake_build
|
| 127 |
+
run: |
|
| 128 |
+
cmake -B build -S . \
|
| 129 |
+
-DCMAKE_HIP_COMPILER="$(hipconfig -l)/clang" \
|
| 130 |
+
-DGPU_TARGETS="gfx1030" \
|
| 131 |
+
-DGGML_HIP=ON
|
| 132 |
+
cmake --build build --config Release -j $(nproc)
|
| 133 |
+
|
| 134 |
+
- name: ccache-buckets-save
|
| 135 |
+
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
| 136 |
+
uses: ./.github/actions/ccache-buckets
|
| 137 |
+
env:
|
| 138 |
+
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
| 139 |
+
with:
|
| 140 |
+
key: cuda-ubuntu-22.04-hip
|
| 141 |
+
folder: llama.cpp
|
| 142 |
+
evict-old-files: 1d
|
| 143 |
+
hf_bucket: ggml-org/cache
|
| 144 |
+
save: true
|
| 145 |
+
|
| 146 |
+
musa:
|
| 147 |
+
runs-on: ubuntu-22.04
|
| 148 |
+
container: mthreads/musa:rc4.3.0-devel-ubuntu22.04-amd64
|
| 149 |
+
|
| 150 |
+
steps:
|
| 151 |
+
- name: Clone
|
| 152 |
+
id: checkout
|
| 153 |
+
uses: actions/checkout@v6
|
| 154 |
+
|
| 155 |
+
- name: Dependencies
|
| 156 |
+
id: depends
|
| 157 |
+
run: |
|
| 158 |
+
apt-get update
|
| 159 |
+
apt-get install -y build-essential git cmake libssl-dev jq
|
| 160 |
+
|
| 161 |
+
- name: ccache
|
| 162 |
+
uses: ggml-org/ccache-action@v1.2.24
|
| 163 |
+
with:
|
| 164 |
+
key: cuda-ubuntu-22.04-musa
|
| 165 |
+
save: false
|
| 166 |
+
|
| 167 |
+
- name: ccache-buckets-restore
|
| 168 |
+
uses: ./.github/actions/ccache-buckets
|
| 169 |
+
env:
|
| 170 |
+
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
| 171 |
+
with:
|
| 172 |
+
key: cuda-ubuntu-22.04-musa
|
| 173 |
+
folder: llama.cpp
|
| 174 |
+
hf_bucket: ggml-org/cache
|
| 175 |
+
|
| 176 |
+
- name: Build with native CMake MUSA support
|
| 177 |
+
id: cmake_build
|
| 178 |
+
run: |
|
| 179 |
+
cmake -B build -S . \
|
| 180 |
+
-DGGML_MUSA=ON
|
| 181 |
+
time cmake --build build --config Release -j $(nproc)
|
| 182 |
+
|
| 183 |
+
- name: ccache-buckets-save
|
| 184 |
+
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
| 185 |
+
uses: ./.github/actions/ccache-buckets
|
| 186 |
+
env:
|
| 187 |
+
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
| 188 |
+
with:
|
| 189 |
+
key: cuda-ubuntu-22.04-musa
|
| 190 |
+
folder: llama.cpp
|
| 191 |
+
evict-old-files: 1d
|
| 192 |
+
hf_bucket: ggml-org/cache
|
| 193 |
+
save: true
|
.github/workflows/build-cuda-windows.yml
ADDED
|
@@ -0,0 +1,183 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: CI (CUDA, windows)
|
| 2 |
+
|
| 3 |
+
# TODO: this workflow is only triggered manually because it is very heavy on the CI
|
| 4 |
+
# when we provision dedicated windows runners, we can enable it for pushes too
|
| 5 |
+
# note: running this workflow manually will populate the ccache for the release builds
|
| 6 |
+
# this can be used before merging a PR to speed up the release workflow
|
| 7 |
+
on:
|
| 8 |
+
workflow_dispatch: # allows manual triggering
|
| 9 |
+
|
| 10 |
+
# note: this will run in queue with the release workflow
|
| 11 |
+
concurrency:
|
| 12 |
+
group: release
|
| 13 |
+
queue: max
|
| 14 |
+
|
| 15 |
+
env:
|
| 16 |
+
GH_TOKEN: ${{ github.token }}
|
| 17 |
+
GGML_NLOOP: 3
|
| 18 |
+
GGML_N_THREADS: 1
|
| 19 |
+
LLAMA_ARG_LOG_COLORS: 1
|
| 20 |
+
LLAMA_ARG_LOG_PREFIX: 1
|
| 21 |
+
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
| 22 |
+
|
| 23 |
+
jobs:
|
| 24 |
+
cuda:
|
| 25 |
+
name: windows-cuda (${{ matrix.cuda }}, ${{ matrix.arch }})
|
| 26 |
+
runs-on: windows-2022
|
| 27 |
+
|
| 28 |
+
permissions:
|
| 29 |
+
actions: write
|
| 30 |
+
|
| 31 |
+
strategy:
|
| 32 |
+
matrix:
|
| 33 |
+
include:
|
| 34 |
+
- cuda: '12.4'
|
| 35 |
+
arch: x64
|
| 36 |
+
defines: '-DGGML_CUDA_CUB_3DOT2=ON'
|
| 37 |
+
- cuda: '13.3'
|
| 38 |
+
arch: x64
|
| 39 |
+
defines: ''
|
| 40 |
+
- cuda: '13.4'
|
| 41 |
+
arch: arm64
|
| 42 |
+
defines: '-DCMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-msvc-cuda.cmake'
|
| 43 |
+
|
| 44 |
+
steps:
|
| 45 |
+
- name: Clone
|
| 46 |
+
id: checkout
|
| 47 |
+
uses: actions/checkout@v6
|
| 48 |
+
|
| 49 |
+
- name: ccache
|
| 50 |
+
uses: ggml-org/ccache-action@v1.2.24
|
| 51 |
+
with:
|
| 52 |
+
key: release-windows-2022-${{ matrix.arch }}-cuda-${{ matrix.cuda }}
|
| 53 |
+
|
| 54 |
+
- name: Install Cuda Toolkit
|
| 55 |
+
uses: ./.github/actions/windows-setup-cuda
|
| 56 |
+
with:
|
| 57 |
+
cuda_version: ${{ matrix.cuda }}
|
| 58 |
+
cuda_arch: ${{ matrix.arch }}
|
| 59 |
+
|
| 60 |
+
- name: Install Ninja
|
| 61 |
+
id: install_ninja
|
| 62 |
+
run: |
|
| 63 |
+
choco install ninja
|
| 64 |
+
|
| 65 |
+
- name: Build
|
| 66 |
+
id: cmake_build
|
| 67 |
+
shell: cmd
|
| 68 |
+
run: |
|
| 69 |
+
call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvarsall.bat" ${{ matrix.arch == 'x64' && 'x64' || 'amd64_arm64' }}
|
| 70 |
+
cmake -S . -B build -G "Ninja Multi-Config" ^
|
| 71 |
+
-DGGML_BACKEND_DL=ON ^
|
| 72 |
+
-DGGML_NATIVE=OFF ^
|
| 73 |
+
-DGGML_CPU=OFF ^
|
| 74 |
+
-DGGML_CUDA=ON ^
|
| 75 |
+
-DLLAMA_BUILD_BORINGSSL=ON ${{ matrix.defines }}
|
| 76 |
+
set /A NINJA_JOBS=%NUMBER_OF_PROCESSORS%-1
|
| 77 |
+
cmake --build build --config Release -j %NINJA_JOBS% --target ggml-cuda
|
| 78 |
+
|
| 79 |
+
- name: ccache-clear
|
| 80 |
+
uses: ./.github/actions/ccache-clear
|
| 81 |
+
with:
|
| 82 |
+
key: release-windows-2022-${{ matrix.arch }}-cuda-${{ matrix.cuda }}
|
| 83 |
+
|
| 84 |
+
hip:
|
| 85 |
+
runs-on: windows-2022
|
| 86 |
+
|
| 87 |
+
permissions:
|
| 88 |
+
actions: write
|
| 89 |
+
|
| 90 |
+
env:
|
| 91 |
+
# Make sure this is in sync with build-cache.yml
|
| 92 |
+
ROCM_VERSION: "7.14.0"
|
| 93 |
+
|
| 94 |
+
strategy:
|
| 95 |
+
matrix:
|
| 96 |
+
include:
|
| 97 |
+
# sync with release.yml
|
| 98 |
+
- name: "radeon"
|
| 99 |
+
gpu_targets: "gfx1150;gfx1151;gfx1200;gfx1201;gfx1100;gfx1101;gfx1102;gfx1030;gfx1031;gfx1032"
|
| 100 |
+
|
| 101 |
+
steps:
|
| 102 |
+
- name: Clone
|
| 103 |
+
id: checkout
|
| 104 |
+
uses: actions/checkout@v6
|
| 105 |
+
|
| 106 |
+
# - name: Cache ROCm Installation
|
| 107 |
+
# uses: actions/cache@v5
|
| 108 |
+
# id: cache-rocm
|
| 109 |
+
# with:
|
| 110 |
+
# path: C:\TheRock\build
|
| 111 |
+
# key: rocm-wheels-${{ env.ROCM_VERSION }}-multi-arch-${{ runner.os }}
|
| 112 |
+
|
| 113 |
+
- name: Setup ROCm
|
| 114 |
+
# if: steps.cache-rocm.outputs.cache-hit != 'true'
|
| 115 |
+
uses: ./.github/actions/windows-setup-rocm
|
| 116 |
+
with:
|
| 117 |
+
version: ${{ env.ROCM_VERSION }}
|
| 118 |
+
|
| 119 |
+
- name: Setup ROCm Environment
|
| 120 |
+
run: |
|
| 121 |
+
$ErrorActionPreference = "Stop"
|
| 122 |
+
|
| 123 |
+
# Activate venv from cache or fresh install
|
| 124 |
+
& C:\TheRock\build\.venv\Scripts\Activate.ps1
|
| 125 |
+
|
| 126 |
+
# Expand the devel tree (idempotent; no-op if already done during install)
|
| 127 |
+
rocm-sdk init
|
| 128 |
+
if ($LASTEXITCODE -ne 0) { throw "rocm-sdk init failed with exit code $LASTEXITCODE" }
|
| 129 |
+
|
| 130 |
+
# Get ROCm installation paths using the rocm-sdk CLI tool
|
| 131 |
+
$rocmPath = (rocm-sdk path --root)
|
| 132 |
+
if (-not $rocmPath) { throw "rocm-sdk path --root returned empty - devel package may not be installed" }
|
| 133 |
+
$rocmPath = $rocmPath.Trim()
|
| 134 |
+
$cmakePath = (rocm-sdk path --cmake).Trim()
|
| 135 |
+
$binPath = (rocm-sdk path --bin).Trim()
|
| 136 |
+
write-host "ROCm root: $rocmPath"
|
| 137 |
+
|
| 138 |
+
echo "HIP_PATH=$rocmPath" >> $env:GITHUB_ENV
|
| 139 |
+
echo "CMAKE_PREFIX_PATH=$cmakePath" >> $env:GITHUB_ENV
|
| 140 |
+
echo "HIP_DEVICE_LIB_PATH=$rocmPath\lib\llvm\amdgcn\bitcode" >> $env:GITHUB_ENV
|
| 141 |
+
echo "HIP_PLATFORM=amd" >> $env:GITHUB_ENV
|
| 142 |
+
echo "LLVM_PATH=$rocmPath\lib\llvm" >> $env:GITHUB_ENV
|
| 143 |
+
echo "$binPath" >> $env:GITHUB_PATH
|
| 144 |
+
|
| 145 |
+
# Keep venv in PATH for subsequent steps
|
| 146 |
+
echo "C:\TheRock\build\.venv\Scripts" >> $env:GITHUB_PATH
|
| 147 |
+
|
| 148 |
+
- name: Verify ROCm
|
| 149 |
+
id: verify
|
| 150 |
+
run: |
|
| 151 |
+
# Test the ROCm clang shipped in the installed wheel
|
| 152 |
+
& "${env:HIP_PATH}\lib\llvm\bin\clang.exe" --version
|
| 153 |
+
|
| 154 |
+
- name: ccache
|
| 155 |
+
uses: ggml-org/ccache-action@v1.2.24
|
| 156 |
+
with:
|
| 157 |
+
# TODO: this build does not match the build in release.yml, so we use a different cache key
|
| 158 |
+
# ideally, the builds should match, similar to the CUDA build above so that we would be able
|
| 159 |
+
# to populate the ccache for the release with manual runs of this workflow
|
| 160 |
+
#key: release-windows-2022-x64-hip-${{ env.ROCM_VERSION }}-${{ matrix.name }}
|
| 161 |
+
key: cuda-windows-2022-x64-hip-${{ env.ROCM_VERSION }}-${{ matrix.name }}
|
| 162 |
+
|
| 163 |
+
- name: Build
|
| 164 |
+
id: cmake_build
|
| 165 |
+
run: |
|
| 166 |
+
cmake -G "Unix Makefiles" -B build -S . `
|
| 167 |
+
-DCMAKE_PREFIX_PATH="${env:HIP_PATH}" `
|
| 168 |
+
-DCMAKE_C_COMPILER="${env:HIP_PATH}\lib\llvm\bin\clang.exe" `
|
| 169 |
+
-DCMAKE_CXX_COMPILER="${env:HIP_PATH}\lib\llvm\bin\clang++.exe" `
|
| 170 |
+
-DCMAKE_HIP_COMPILER="${env:HIP_PATH}\lib\llvm\bin\clang.exe" `
|
| 171 |
+
-DCMAKE_BUILD_TYPE=Release `
|
| 172 |
+
-DLLAMA_BUILD_BORINGSSL=ON `
|
| 173 |
+
-DHIP_PATH="${env:HIP_PATH}" `
|
| 174 |
+
-DGGML_HIP=ON `
|
| 175 |
+
-DGPU_TARGETS="gfx1100" `
|
| 176 |
+
-DGGML_RPC=ON
|
| 177 |
+
cmake --build build -j ${env:NUMBER_OF_PROCESSORS}
|
| 178 |
+
|
| 179 |
+
- name: ccache-clear
|
| 180 |
+
uses: ./.github/actions/ccache-clear
|
| 181 |
+
with:
|
| 182 |
+
#key: release-windows-2022-x64-hip-${{ env.ROCM_VERSION }}-${{ matrix.name }}
|
| 183 |
+
key: cuda-windows-2022-x64-hip-${{ env.ROCM_VERSION }}-${{ matrix.name }}
|
.github/workflows/build-ibm.yml
ADDED
|
@@ -0,0 +1,150 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: CI (ibm)
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
workflow_dispatch: # allows manual triggering
|
| 5 |
+
push:
|
| 6 |
+
branches:
|
| 7 |
+
- master
|
| 8 |
+
paths: [
|
| 9 |
+
'.github/workflows/build-ibm.yml',
|
| 10 |
+
'**/CMakeLists.txt',
|
| 11 |
+
'**/.cmake',
|
| 12 |
+
'**/*.h',
|
| 13 |
+
'**/*.hpp',
|
| 14 |
+
'**/*.c',
|
| 15 |
+
'**/*.cpp'
|
| 16 |
+
]
|
| 17 |
+
|
| 18 |
+
pull_request:
|
| 19 |
+
types: [opened, synchronize, reopened]
|
| 20 |
+
paths: [
|
| 21 |
+
'.github/workflows/build-ibm.yml',
|
| 22 |
+
'ggml/src/ggml-cpu/**'
|
| 23 |
+
]
|
| 24 |
+
|
| 25 |
+
concurrency:
|
| 26 |
+
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
| 27 |
+
cancel-in-progress: true
|
| 28 |
+
|
| 29 |
+
env:
|
| 30 |
+
GGML_NLOOP: 3
|
| 31 |
+
GGML_N_THREADS: 1
|
| 32 |
+
LLAMA_ARG_LOG_COLORS: 1
|
| 33 |
+
LLAMA_ARG_LOG_PREFIX: 1
|
| 34 |
+
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
| 35 |
+
|
| 36 |
+
jobs:
|
| 37 |
+
|
| 38 |
+
ubuntu-24-s390x:
|
| 39 |
+
runs-on: ubuntu-24.04-s390x
|
| 40 |
+
|
| 41 |
+
steps:
|
| 42 |
+
- name: Clone
|
| 43 |
+
id: checkout
|
| 44 |
+
uses: actions/checkout@v6
|
| 45 |
+
|
| 46 |
+
- name: Build Dependencies
|
| 47 |
+
id: build_depends
|
| 48 |
+
run: |
|
| 49 |
+
sudo apt-get update
|
| 50 |
+
sudo apt-get install -y --no-install-recommends \
|
| 51 |
+
python3 python3-pip python3-dev python3-wheel \
|
| 52 |
+
libjpeg-dev build-essential libssl-dev \
|
| 53 |
+
git-lfs
|
| 54 |
+
|
| 55 |
+
- name: Toolchain workaround (GCC 14)
|
| 56 |
+
run: |
|
| 57 |
+
sudo apt-get install -y gcc-14 g++-14
|
| 58 |
+
echo "CC=gcc-14" >> "$GITHUB_ENV"
|
| 59 |
+
echo "CXX=g++-14" >> "$GITHUB_ENV"
|
| 60 |
+
|
| 61 |
+
- name: Python Dependencies
|
| 62 |
+
id: python_depends
|
| 63 |
+
run: |
|
| 64 |
+
export PIP_BREAK_SYSTEM_PACKAGES="1"
|
| 65 |
+
python3 -m pip install --upgrade pip setuptools
|
| 66 |
+
pip3 install ./gguf-py
|
| 67 |
+
|
| 68 |
+
- name: Swap Endianness
|
| 69 |
+
id: endianness
|
| 70 |
+
run: |
|
| 71 |
+
for f in models/*.gguf; do
|
| 72 |
+
echo YES | python3 gguf-py/gguf/scripts/gguf_convert_endian.py $f big
|
| 73 |
+
done
|
| 74 |
+
|
| 75 |
+
- name: Build
|
| 76 |
+
id: cmake_build
|
| 77 |
+
run: |
|
| 78 |
+
cmake -B build \
|
| 79 |
+
-DLLAMA_FATAL_WARNINGS=ON \
|
| 80 |
+
-DGGML_RPC=ON
|
| 81 |
+
time cmake --build build --config Release -j $(nproc)
|
| 82 |
+
|
| 83 |
+
- name: Test
|
| 84 |
+
id: cmake_test
|
| 85 |
+
run: |
|
| 86 |
+
cd build
|
| 87 |
+
ctest -L main --verbose --timeout 900
|
| 88 |
+
|
| 89 |
+
- name: Test llama2c (s390x)
|
| 90 |
+
id: llama2c_test_s390x
|
| 91 |
+
run: |
|
| 92 |
+
cd build
|
| 93 |
+
echo "Fetch llama2c big-endian model"
|
| 94 |
+
wget https://huggingface.co/ggml-org/models/resolve/main/tinyllamas/stories260K-be.gguf
|
| 95 |
+
./bin/llama-completion -m stories260K-be.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256
|
| 96 |
+
|
| 97 |
+
ubuntu-24-ppc64le:
|
| 98 |
+
runs-on: ubuntu-24.04-ppc64le
|
| 99 |
+
|
| 100 |
+
steps:
|
| 101 |
+
- name: Clone
|
| 102 |
+
id: checkout
|
| 103 |
+
uses: actions/checkout@v6
|
| 104 |
+
|
| 105 |
+
- name: Build Dependencies
|
| 106 |
+
id: build_depends
|
| 107 |
+
run: |
|
| 108 |
+
sudo apt-get update
|
| 109 |
+
sudo apt-get install -y --no-install-recommends \
|
| 110 |
+
python3 python3-pip python3-dev python3-wheel \
|
| 111 |
+
libjpeg-dev build-essential libssl-dev \
|
| 112 |
+
git-lfs
|
| 113 |
+
|
| 114 |
+
- name: Toolchain workaround (GCC 14)
|
| 115 |
+
run: |
|
| 116 |
+
sudo apt-get install -y gcc-14 g++-14
|
| 117 |
+
echo "CC=gcc-14" >> "$GITHUB_ENV"
|
| 118 |
+
echo "CXX=g++-14" >> "$GITHUB_ENV"
|
| 119 |
+
|
| 120 |
+
- name: Python Dependencies
|
| 121 |
+
id: python_depends
|
| 122 |
+
run: |
|
| 123 |
+
export PIP_BREAK_SYSTEM_PACKAGES="1"
|
| 124 |
+
python3 -m pip install --upgrade pip setuptools
|
| 125 |
+
pip3 install ./gguf-py
|
| 126 |
+
|
| 127 |
+
- name: Build
|
| 128 |
+
id: cmake_build
|
| 129 |
+
run: |
|
| 130 |
+
cmake -B build \
|
| 131 |
+
-DLLAMA_FATAL_WARNINGS=ON \
|
| 132 |
+
-DGGML_RPC=ON
|
| 133 |
+
time cmake --build build --config Release -j $(nproc)
|
| 134 |
+
|
| 135 |
+
- name: Test
|
| 136 |
+
id: cmake_test
|
| 137 |
+
run: |
|
| 138 |
+
cd build
|
| 139 |
+
ctest -L main --verbose --timeout 900
|
| 140 |
+
|
| 141 |
+
- name: Test llama2c conversion
|
| 142 |
+
id: llama2c_test
|
| 143 |
+
run: |
|
| 144 |
+
cd build
|
| 145 |
+
echo "Fetch tokenizer"
|
| 146 |
+
wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/tok512.bin
|
| 147 |
+
echo "Fetch llama2c model"
|
| 148 |
+
wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/stories260K.bin
|
| 149 |
+
./bin/llama-convert-llama2c-to-ggml --copy-vocab-from-model ./tok512.bin --llama2c-model stories260K.bin --llama2c-output-model stories260K.gguf
|
| 150 |
+
./bin/llama-completion -m stories260K.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256
|
.github/workflows/build-msys.yml
ADDED
|
@@ -0,0 +1,70 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: CI (msys)
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
# only manual triggers due to low-importance of the workflows
|
| 5 |
+
# TODO: for regular runs, provision dedicated self-hosted runners
|
| 6 |
+
workflow_dispatch:
|
| 7 |
+
# run once every week
|
| 8 |
+
schedule:
|
| 9 |
+
- cron: '0 0 * * 0'
|
| 10 |
+
|
| 11 |
+
concurrency:
|
| 12 |
+
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
| 13 |
+
cancel-in-progress: true
|
| 14 |
+
|
| 15 |
+
env:
|
| 16 |
+
GGML_NLOOP: 3
|
| 17 |
+
GGML_N_THREADS: 1
|
| 18 |
+
LLAMA_ARG_LOG_COLORS: 1
|
| 19 |
+
LLAMA_ARG_LOG_PREFIX: 1
|
| 20 |
+
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
| 21 |
+
|
| 22 |
+
jobs:
|
| 23 |
+
windows-msys2:
|
| 24 |
+
runs-on: windows-2025
|
| 25 |
+
|
| 26 |
+
strategy:
|
| 27 |
+
fail-fast: false
|
| 28 |
+
matrix:
|
| 29 |
+
include:
|
| 30 |
+
- { sys: UCRT64, env: ucrt-x86_64, compiler: gcc, build: Release }
|
| 31 |
+
- { sys: CLANG64, env: clang-x86_64, compiler: clang, build: Release }
|
| 32 |
+
|
| 33 |
+
steps:
|
| 34 |
+
- name: Clone
|
| 35 |
+
uses: actions/checkout@v6
|
| 36 |
+
|
| 37 |
+
#- name: ccache
|
| 38 |
+
# uses: ggml-org/ccache-action@v1.2.24
|
| 39 |
+
# with:
|
| 40 |
+
# key: msys-windows-2025-x64
|
| 41 |
+
# variant: ccache
|
| 42 |
+
# evict-old-files: 1d
|
| 43 |
+
# save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
| 44 |
+
|
| 45 |
+
- name: Setup ${{ matrix.sys }}
|
| 46 |
+
uses: msys2/setup-msys2@cafece8e6baf9247cf9b1bf95097b0b983cc558d # v2
|
| 47 |
+
with:
|
| 48 |
+
update: true
|
| 49 |
+
msystem: ${{matrix.sys}}
|
| 50 |
+
install: >-
|
| 51 |
+
mingw-w64-${{matrix.env}}-${{matrix.compiler}}
|
| 52 |
+
mingw-w64-${{matrix.env}}-cmake
|
| 53 |
+
mingw-w64-${{matrix.env}}-openblas
|
| 54 |
+
|
| 55 |
+
- name: Build using CMake
|
| 56 |
+
shell: msys2 {0}
|
| 57 |
+
run: |
|
| 58 |
+
cmake -B build
|
| 59 |
+
cmake --build build --config ${{ matrix.build }} -j $(nproc)
|
| 60 |
+
|
| 61 |
+
- name: Clean after building using CMake
|
| 62 |
+
shell: msys2 {0}
|
| 63 |
+
run: |
|
| 64 |
+
rm -rf build
|
| 65 |
+
|
| 66 |
+
- name: Build using CMake w/ OpenBLAS
|
| 67 |
+
shell: msys2 {0}
|
| 68 |
+
run: |
|
| 69 |
+
cmake -B build -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS
|
| 70 |
+
cmake --build build --config ${{ matrix.build }} -j $(nproc)
|
.github/workflows/build-opencl.yml
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: CI (opencl)
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
workflow_dispatch: # allows manual triggering
|
| 5 |
+
push:
|
| 6 |
+
branches:
|
| 7 |
+
- master
|
| 8 |
+
paths: [
|
| 9 |
+
'.github/workflows/build-opencl.yml',
|
| 10 |
+
'**/CMakeLists.txt',
|
| 11 |
+
'**/.cmake',
|
| 12 |
+
'**/*.h',
|
| 13 |
+
'**/*.hpp',
|
| 14 |
+
'**/*.c',
|
| 15 |
+
'**/*.cpp',
|
| 16 |
+
'**/*.cl'
|
| 17 |
+
]
|
| 18 |
+
|
| 19 |
+
pull_request:
|
| 20 |
+
types: [opened, synchronize, reopened]
|
| 21 |
+
paths: [
|
| 22 |
+
'.github/workflows/build-opencl.yml',
|
| 23 |
+
'ggml/src/ggml-opencl/**'
|
| 24 |
+
]
|
| 25 |
+
|
| 26 |
+
concurrency:
|
| 27 |
+
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
| 28 |
+
cancel-in-progress: true
|
| 29 |
+
|
| 30 |
+
env:
|
| 31 |
+
GGML_NLOOP: 3
|
| 32 |
+
GGML_N_THREADS: 1
|
| 33 |
+
LLAMA_ARG_LOG_COLORS: 1
|
| 34 |
+
LLAMA_ARG_LOG_PREFIX: 1
|
| 35 |
+
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
| 36 |
+
|
| 37 |
+
jobs:
|
| 38 |
+
windows-2025-opencl-adreno:
|
| 39 |
+
runs-on: windows-2025
|
| 40 |
+
|
| 41 |
+
steps:
|
| 42 |
+
- name: Clone
|
| 43 |
+
id: checkout
|
| 44 |
+
uses: actions/checkout@v6
|
| 45 |
+
|
| 46 |
+
- name: ccache
|
| 47 |
+
uses: ggml-org/ccache-action@v1.2.24
|
| 48 |
+
with:
|
| 49 |
+
key: opencl-windows-2025-x64
|
| 50 |
+
variant: ccache
|
| 51 |
+
evict-old-files: 1d
|
| 52 |
+
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
| 53 |
+
|
| 54 |
+
- name: Install Ninja
|
| 55 |
+
id: install_ninja
|
| 56 |
+
run: |
|
| 57 |
+
choco install ninja
|
| 58 |
+
|
| 59 |
+
- name: Install OpenCL Headers and Libs
|
| 60 |
+
id: install_opencl
|
| 61 |
+
run: |
|
| 62 |
+
git clone https://github.com/KhronosGroup/OpenCL-Headers
|
| 63 |
+
cd OpenCL-Headers
|
| 64 |
+
cmake -B build `
|
| 65 |
+
-DBUILD_TESTING=OFF `
|
| 66 |
+
-DOPENCL_HEADERS_BUILD_TESTING=OFF `
|
| 67 |
+
-DOPENCL_HEADERS_BUILD_CXX_TESTS=OFF `
|
| 68 |
+
-DCMAKE_INSTALL_PREFIX="$env:RUNNER_TEMP/opencl-arm64-release"
|
| 69 |
+
cmake --build build --target install
|
| 70 |
+
git clone https://github.com/KhronosGroup/OpenCL-ICD-Loader
|
| 71 |
+
cd OpenCL-ICD-Loader
|
| 72 |
+
cmake -B build-arm64-release `
|
| 73 |
+
-A arm64 `
|
| 74 |
+
-DCMAKE_PREFIX_PATH="$env:RUNNER_TEMP/opencl-arm64-release" `
|
| 75 |
+
-DCMAKE_INSTALL_PREFIX="$env:RUNNER_TEMP/opencl-arm64-release"
|
| 76 |
+
cmake --build build-arm64-release --target install --config release
|
| 77 |
+
|
| 78 |
+
- name: Build
|
| 79 |
+
id: cmake_build
|
| 80 |
+
run: |
|
| 81 |
+
cmake -S . -B build -G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DCMAKE_PREFIX_PATH="$env:RUNNER_TEMP/opencl-arm64-release" -DGGML_OPENCL=ON -DGGML_OPENCL_USE_ADRENO_KERNELS=ON -DLLAMA_BUILD_BORINGSSL=ON
|
| 82 |
+
cmake --build build --config Release -j ${env:NUMBER_OF_PROCESSORS}
|
| 83 |
+
|
| 84 |
+
- name: ccache-clear
|
| 85 |
+
uses: ./.github/actions/ccache-clear
|
| 86 |
+
env:
|
| 87 |
+
GH_TOKEN: ${{ github.token }}
|
| 88 |
+
with:
|
| 89 |
+
key: opencl-windows-2025-x64
|
| 90 |
+
older: 5m
|
| 91 |
+
min: 1
|
| 92 |
+
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
|
.github/workflows/build-openvino.yml
ADDED
|
@@ -0,0 +1,178 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: CI (openvino)
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
workflow_dispatch: # allows manual triggering
|
| 5 |
+
push:
|
| 6 |
+
branches:
|
| 7 |
+
- master
|
| 8 |
+
paths: [
|
| 9 |
+
'.github/workflows/build-openvino.yml',
|
| 10 |
+
'**/CMakeLists.txt',
|
| 11 |
+
'**/.cmake',
|
| 12 |
+
'**/*.h',
|
| 13 |
+
'**/*.hpp',
|
| 14 |
+
'**/*.c',
|
| 15 |
+
'**/*.cpp',
|
| 16 |
+
]
|
| 17 |
+
|
| 18 |
+
pull_request:
|
| 19 |
+
types: [opened, synchronize, reopened]
|
| 20 |
+
paths: [
|
| 21 |
+
'.github/workflows/build-openvino.yml',
|
| 22 |
+
'ggml/src/ggml-openvino/**'
|
| 23 |
+
]
|
| 24 |
+
|
| 25 |
+
concurrency:
|
| 26 |
+
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
| 27 |
+
cancel-in-progress: true
|
| 28 |
+
|
| 29 |
+
env:
|
| 30 |
+
GGML_NLOOP: 3
|
| 31 |
+
GGML_N_THREADS: 1
|
| 32 |
+
LLAMA_ARG_LOG_COLORS: 1
|
| 33 |
+
LLAMA_ARG_LOG_PREFIX: 1
|
| 34 |
+
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
| 35 |
+
# TODO: fix failing tests on OpenVINO backend
|
| 36 |
+
CTEST_EXCLUDE: "test-llama-archs|^test-recurrent-state-|test-backend-ops|test-save-load-state"
|
| 37 |
+
|
| 38 |
+
jobs:
|
| 39 |
+
ubuntu-24-openvino:
|
| 40 |
+
runs-on: [self-hosted, Linux, Intel, OpenVINO]
|
| 41 |
+
|
| 42 |
+
env:
|
| 43 |
+
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
|
| 44 |
+
OPENVINO_VERSION_MAJOR: "2026.3.1"
|
| 45 |
+
OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
|
| 46 |
+
|
| 47 |
+
steps:
|
| 48 |
+
- name: Clone
|
| 49 |
+
id: checkout
|
| 50 |
+
uses: actions/checkout@v6
|
| 51 |
+
|
| 52 |
+
- name: Dependencies
|
| 53 |
+
id: depends
|
| 54 |
+
run: |
|
| 55 |
+
sudo apt-get update
|
| 56 |
+
sudo apt-get install -y build-essential libssl-dev libtbb12 cmake ninja-build python3-pip
|
| 57 |
+
sudo apt-get install -y ocl-icd-opencl-dev opencl-headers opencl-clhpp-headers intel-opencl-icd
|
| 58 |
+
|
| 59 |
+
- name: Setup OpenVINO Toolkit
|
| 60 |
+
uses: ./.github/actions/linux-setup-openvino
|
| 61 |
+
with:
|
| 62 |
+
path: ./openvino_toolkit
|
| 63 |
+
version_major: ${{ env.OPENVINO_VERSION_MAJOR }}
|
| 64 |
+
version_full: ${{ env.OPENVINO_VERSION_FULL }}
|
| 65 |
+
|
| 66 |
+
- name: Install OpenVINO dependencies
|
| 67 |
+
run: |
|
| 68 |
+
cd ./openvino_toolkit
|
| 69 |
+
chmod +x ./install_dependencies/install_openvino_dependencies.sh
|
| 70 |
+
echo "Y" | sudo -E ./install_dependencies/install_openvino_dependencies.sh
|
| 71 |
+
|
| 72 |
+
- name: Build
|
| 73 |
+
id: cmake_build
|
| 74 |
+
run: |
|
| 75 |
+
source ./openvino_toolkit/setupvars.sh
|
| 76 |
+
cmake -B build/ReleaseOV -G Ninja \
|
| 77 |
+
-DCMAKE_BUILD_TYPE=Release \
|
| 78 |
+
-DGGML_OPENVINO=ON
|
| 79 |
+
time cmake --build build/ReleaseOV --config Release --parallel
|
| 80 |
+
|
| 81 |
+
- name: Test (CPU)
|
| 82 |
+
id: cmake_test_cpu
|
| 83 |
+
run: |
|
| 84 |
+
cd ${{ github.workspace }}
|
| 85 |
+
ctest --test-dir build/ReleaseOV -L main -E "${{ env.CTEST_EXCLUDE }}" --verbose --timeout 3000
|
| 86 |
+
|
| 87 |
+
- name: Test (GPU)
|
| 88 |
+
id: cmake_test_gpu
|
| 89 |
+
run: |
|
| 90 |
+
cd ${{ github.workspace }}
|
| 91 |
+
export GGML_OPENVINO_DEVICE=GPU
|
| 92 |
+
ctest --test-dir build/ReleaseOV -L main -E "${{ env.CTEST_EXCLUDE }}" --verbose --timeout 3000
|
| 93 |
+
|
| 94 |
+
openvino-windows-2022:
|
| 95 |
+
runs-on: windows-2022
|
| 96 |
+
|
| 97 |
+
env:
|
| 98 |
+
# Sync versions in build-openvino.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
|
| 99 |
+
OPENVINO_VERSION_MAJOR: "2026.3.1"
|
| 100 |
+
OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
|
| 101 |
+
|
| 102 |
+
steps:
|
| 103 |
+
- name: Clone
|
| 104 |
+
id: checkout
|
| 105 |
+
uses: actions/checkout@v6
|
| 106 |
+
|
| 107 |
+
- name: ccache
|
| 108 |
+
uses: ggml-org/ccache-action@v1.2.24
|
| 109 |
+
with:
|
| 110 |
+
key: openvino-windows-2022
|
| 111 |
+
variant: ccache
|
| 112 |
+
evict-old-files: 1d
|
| 113 |
+
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
| 114 |
+
|
| 115 |
+
- name: Setup Cache
|
| 116 |
+
uses: actions/cache@v5
|
| 117 |
+
id: cache-openvino
|
| 118 |
+
with:
|
| 119 |
+
path: ./openvino_toolkit
|
| 120 |
+
key: cache-gha-openvino-toolkit-v${{ env.OPENVINO_VERSION_FULL }}-${{ runner.os }}
|
| 121 |
+
|
| 122 |
+
- name: Setup OpenVINO Toolkit
|
| 123 |
+
if: steps.cache-openvino.outputs.cache-hit != 'true'
|
| 124 |
+
uses: ./.github/actions/windows-setup-openvino
|
| 125 |
+
with:
|
| 126 |
+
path: ./openvino_toolkit
|
| 127 |
+
version_major: ${{ env.OPENVINO_VERSION_MAJOR }}
|
| 128 |
+
version_full: ${{ env.OPENVINO_VERSION_FULL }}
|
| 129 |
+
|
| 130 |
+
- name: Install OpenCL using vcpkg
|
| 131 |
+
shell: powershell
|
| 132 |
+
run: |
|
| 133 |
+
git clone https://github.com/microsoft/vcpkg C:\vcpkg
|
| 134 |
+
C:\vcpkg\bootstrap-vcpkg.bat
|
| 135 |
+
C:\vcpkg\vcpkg install opencl
|
| 136 |
+
|
| 137 |
+
- name: Build
|
| 138 |
+
id: cmake_build
|
| 139 |
+
shell: cmd
|
| 140 |
+
run: |
|
| 141 |
+
REM Find extracted OpenVINO folder dynamically
|
| 142 |
+
for /d %%i in (openvino_toolkit\*) do set OPENVINO_ROOT=%%i
|
| 143 |
+
|
| 144 |
+
if not exist "%OPENVINO_ROOT%\runtime\cmake\OpenVINOConfig.cmake" (
|
| 145 |
+
echo ERROR: OpenVINOConfig.cmake not found
|
| 146 |
+
exit /b 1
|
| 147 |
+
)
|
| 148 |
+
|
| 149 |
+
call "%OPENVINO_ROOT%\setupvars.bat"
|
| 150 |
+
|
| 151 |
+
cmake -B build\ReleaseOV -G "Visual Studio 17 2022" ^
|
| 152 |
+
-A x64 ^
|
| 153 |
+
-DCMAKE_BUILD_TYPE=Release ^
|
| 154 |
+
-DGGML_OPENVINO=ON ^
|
| 155 |
+
-DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake
|
| 156 |
+
|
| 157 |
+
cmake --build build\ReleaseOV --config Release -- /m
|
| 158 |
+
|
| 159 |
+
- name: Test (CPU)
|
| 160 |
+
id: cmake_test_cpu
|
| 161 |
+
shell: cmd
|
| 162 |
+
run: |
|
| 163 |
+
REM Find extracted OpenVINO folder dynamically
|
| 164 |
+
for /d %%i in (openvino_toolkit\*) do set OPENVINO_ROOT=%%i
|
| 165 |
+
call "%OPENVINO_ROOT%\setupvars.bat"
|
| 166 |
+
|
| 167 |
+
cd build
|
| 168 |
+
ctest --test-dir ReleaseOV -L main -E "${{ env.CTEST_EXCLUDE }}" -C Release --verbose --timeout 3000
|
| 169 |
+
|
| 170 |
+
- name: ccache-clear
|
| 171 |
+
uses: ./.github/actions/ccache-clear
|
| 172 |
+
env:
|
| 173 |
+
GH_TOKEN: ${{ github.token }}
|
| 174 |
+
with:
|
| 175 |
+
key: openvino-windows-2022
|
| 176 |
+
older: 5m
|
| 177 |
+
min: 1
|
| 178 |
+
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
|
.github/workflows/build-riscv.yml
ADDED
|
@@ -0,0 +1,188 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: CI (riscv)
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
workflow_dispatch: # allows manual triggering
|
| 5 |
+
push:
|
| 6 |
+
branches:
|
| 7 |
+
- master
|
| 8 |
+
paths: [
|
| 9 |
+
'.github/workflows/build-riscv.yml',
|
| 10 |
+
'**/CMakeLists.txt',
|
| 11 |
+
'**/.cmake',
|
| 12 |
+
'**/*.h',
|
| 13 |
+
'**/*.hpp',
|
| 14 |
+
'**/*.c',
|
| 15 |
+
'**/*.cpp'
|
| 16 |
+
]
|
| 17 |
+
|
| 18 |
+
pull_request:
|
| 19 |
+
types: [opened, synchronize, reopened]
|
| 20 |
+
paths: [
|
| 21 |
+
'.github/workflows/build-riscv.yml',
|
| 22 |
+
'ggml/src/ggml-cpu/arch/riscv/**'
|
| 23 |
+
]
|
| 24 |
+
|
| 25 |
+
concurrency:
|
| 26 |
+
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
| 27 |
+
cancel-in-progress: true
|
| 28 |
+
|
| 29 |
+
env:
|
| 30 |
+
GGML_NLOOP: 3
|
| 31 |
+
GGML_N_THREADS: 1
|
| 32 |
+
LLAMA_ARG_LOG_COLORS: 1
|
| 33 |
+
LLAMA_ARG_LOG_PREFIX: 1
|
| 34 |
+
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
| 35 |
+
|
| 36 |
+
jobs:
|
| 37 |
+
ubuntu-cpu-riscv64-native:
|
| 38 |
+
runs-on: ubuntu-24.04-riscv
|
| 39 |
+
|
| 40 |
+
steps:
|
| 41 |
+
- name: Install dependencies
|
| 42 |
+
run: |
|
| 43 |
+
# Install necessary packages
|
| 44 |
+
sudo apt-get update
|
| 45 |
+
sudo apt-get install -y libssl-dev
|
| 46 |
+
|
| 47 |
+
# Set gcc-14 and g++-14 as the default compilers
|
| 48 |
+
sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 100
|
| 49 |
+
sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-14 100
|
| 50 |
+
|
| 51 |
+
git lfs install
|
| 52 |
+
|
| 53 |
+
- name: Check environment
|
| 54 |
+
run: |
|
| 55 |
+
uname -a
|
| 56 |
+
gcc --version
|
| 57 |
+
g++ --version
|
| 58 |
+
ldd --version
|
| 59 |
+
cmake --version
|
| 60 |
+
rustc --version
|
| 61 |
+
env
|
| 62 |
+
echo "nproc=$(nproc)"
|
| 63 |
+
|
| 64 |
+
- name: Clone
|
| 65 |
+
id: checkout
|
| 66 |
+
uses: actions/checkout@v6
|
| 67 |
+
|
| 68 |
+
# note: sparing some ccache since these jobs run on dedicated runners that are not part of the organitzation
|
| 69 |
+
#- name: ccache
|
| 70 |
+
# uses: ggml-org/ccache-action@v1.2.24
|
| 71 |
+
# with:
|
| 72 |
+
# key: riscv-ubuntu-native
|
| 73 |
+
# evict-old-files: 1d
|
| 74 |
+
# save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
| 75 |
+
|
| 76 |
+
- name: Build
|
| 77 |
+
id: cmake_build
|
| 78 |
+
run: |
|
| 79 |
+
cmake -B build \
|
| 80 |
+
-DCMAKE_BUILD_TYPE=Release \
|
| 81 |
+
-DGGML_OPENMP=OFF \
|
| 82 |
+
-DLLAMA_BUILD_EXAMPLES=ON \
|
| 83 |
+
-DLLAMA_BUILD_TOOLS=ON \
|
| 84 |
+
-DLLAMA_BUILD_TESTS=ON \
|
| 85 |
+
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
|
| 86 |
+
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
|
| 87 |
+
-DGGML_RPC=ON \
|
| 88 |
+
-DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \
|
| 89 |
+
-DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14
|
| 90 |
+
|
| 91 |
+
time cmake --build build --config Release -j $(nproc)
|
| 92 |
+
|
| 93 |
+
- name: Test
|
| 94 |
+
id: cmake_test
|
| 95 |
+
run: |
|
| 96 |
+
cd build
|
| 97 |
+
ctest -L main --verbose --timeout 900
|
| 98 |
+
|
| 99 |
+
- name: Test llama2c conversion
|
| 100 |
+
id: llama2c_test
|
| 101 |
+
run: |
|
| 102 |
+
cd build
|
| 103 |
+
echo "Fetch tokenizer"
|
| 104 |
+
wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/tok512.bin
|
| 105 |
+
echo "Fetch llama2c model"
|
| 106 |
+
wget https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/stories260K.bin
|
| 107 |
+
./bin/llama-convert-llama2c-to-ggml --copy-vocab-from-model ./tok512.bin --llama2c-model stories260K.bin --llama2c-output-model stories260K.gguf
|
| 108 |
+
./bin/llama-completion -m stories260K.gguf -p "One day, Lily met a Shoggoth" -n 500 -c 256
|
| 109 |
+
|
| 110 |
+
ubuntu-riscv64-native-sanitizer:
|
| 111 |
+
runs-on: ubuntu-24.04-riscv
|
| 112 |
+
|
| 113 |
+
continue-on-error: true
|
| 114 |
+
|
| 115 |
+
strategy:
|
| 116 |
+
matrix:
|
| 117 |
+
sanitizer: [ADDRESS, THREAD, UNDEFINED]
|
| 118 |
+
build_type: [Debug]
|
| 119 |
+
|
| 120 |
+
steps:
|
| 121 |
+
- name: Install dependencies
|
| 122 |
+
run: |
|
| 123 |
+
# Set gcc-14 and g++-14 as the default compilers
|
| 124 |
+
sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 100
|
| 125 |
+
sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-14 100
|
| 126 |
+
|
| 127 |
+
git lfs install
|
| 128 |
+
|
| 129 |
+
- name: GCC version check
|
| 130 |
+
run: |
|
| 131 |
+
gcc --version
|
| 132 |
+
g++ --version
|
| 133 |
+
|
| 134 |
+
- name: Clone
|
| 135 |
+
id: checkout
|
| 136 |
+
uses: actions/checkout@v6
|
| 137 |
+
|
| 138 |
+
# note: sparing some ccache since these jobs run on dedicated runners that are not part of the organitzation
|
| 139 |
+
#- name: ccache
|
| 140 |
+
# uses: ggml-org/ccache-action@v1.2.24
|
| 141 |
+
# with:
|
| 142 |
+
# key: riscv-ubuntu-native-sanitizer-${{ matrix.sanitizer }}-${{ matrix.build_type }}
|
| 143 |
+
# evict-old-files: 1d
|
| 144 |
+
# save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
| 145 |
+
|
| 146 |
+
- name: Build
|
| 147 |
+
id: cmake_build
|
| 148 |
+
if: ${{ matrix.sanitizer != 'THREAD' }}
|
| 149 |
+
run: |
|
| 150 |
+
cmake -B build \
|
| 151 |
+
-DLLAMA_OPENSSL=OFF \
|
| 152 |
+
-DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
|
| 153 |
+
-DGGML_OPENMP=ON \
|
| 154 |
+
-DLLAMA_BUILD_EXAMPLES=ON \
|
| 155 |
+
-DLLAMA_BUILD_TOOLS=ON \
|
| 156 |
+
-DLLAMA_BUILD_TESTS=OFF \
|
| 157 |
+
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
|
| 158 |
+
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
|
| 159 |
+
-DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \
|
| 160 |
+
-DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \
|
| 161 |
+
-DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14
|
| 162 |
+
|
| 163 |
+
cmake --build build --config ${{ matrix.build_type }} -j $(nproc)
|
| 164 |
+
|
| 165 |
+
- name: Build (no OpenMP)
|
| 166 |
+
id: cmake_build_no_openmp
|
| 167 |
+
if: ${{ matrix.sanitizer == 'THREAD' }}
|
| 168 |
+
run: |
|
| 169 |
+
cmake -B build \
|
| 170 |
+
-DLLAMA_OPENSSL=OFF \
|
| 171 |
+
-DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
|
| 172 |
+
-DGGML_OPENMP=OFF \
|
| 173 |
+
-DLLAMA_BUILD_EXAMPLES=ON \
|
| 174 |
+
-DLLAMA_BUILD_TOOLS=ON \
|
| 175 |
+
-DLLAMA_BUILD_TESTS=OFF \
|
| 176 |
+
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
|
| 177 |
+
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
|
| 178 |
+
-DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \
|
| 179 |
+
-DCMAKE_C_COMPILER=riscv64-linux-gnu-gcc-14 \
|
| 180 |
+
-DCMAKE_CXX_COMPILER=riscv64-linux-gnu-g++-14
|
| 181 |
+
|
| 182 |
+
cmake --build build --config ${{ matrix.build_type }} -j $(nproc)
|
| 183 |
+
|
| 184 |
+
- name: Test
|
| 185 |
+
id: cmake_test
|
| 186 |
+
run: |
|
| 187 |
+
cd build
|
| 188 |
+
ctest -L main --verbose --timeout 900
|
.github/workflows/build-sanitize.yml
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: CI (sanitize)
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
workflow_dispatch: # allows manual triggering
|
| 5 |
+
push:
|
| 6 |
+
branches:
|
| 7 |
+
- master
|
| 8 |
+
paths: [
|
| 9 |
+
'.github/workflows/build-sanitize.yml',
|
| 10 |
+
'**/CMakeLists.txt',
|
| 11 |
+
'**/.cmake',
|
| 12 |
+
'**/*.h',
|
| 13 |
+
'**/*.hpp',
|
| 14 |
+
'**/*.c',
|
| 15 |
+
'**/*.cpp'
|
| 16 |
+
]
|
| 17 |
+
|
| 18 |
+
pull_request:
|
| 19 |
+
types: [opened, synchronize, reopened]
|
| 20 |
+
paths: [
|
| 21 |
+
'.github/workflows/build-sanitize.yml'
|
| 22 |
+
]
|
| 23 |
+
|
| 24 |
+
concurrency:
|
| 25 |
+
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
| 26 |
+
cancel-in-progress: true
|
| 27 |
+
|
| 28 |
+
env:
|
| 29 |
+
GGML_NLOOP: 3
|
| 30 |
+
GGML_N_THREADS: 1
|
| 31 |
+
LLAMA_ARG_LOG_COLORS: 1
|
| 32 |
+
LLAMA_ARG_LOG_PREFIX: 1
|
| 33 |
+
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
| 34 |
+
|
| 35 |
+
jobs:
|
| 36 |
+
ctest:
|
| 37 |
+
continue-on-error: true
|
| 38 |
+
|
| 39 |
+
strategy:
|
| 40 |
+
matrix:
|
| 41 |
+
include:
|
| 42 |
+
# thread and address doesn't run properly on some self hosted machines, so run it on Github instead
|
| 43 |
+
- sanitizer: ADDRESS
|
| 44 |
+
machine: ubuntu-24.04
|
| 45 |
+
- sanitizer: THREAD
|
| 46 |
+
machine: ubuntu-24.04
|
| 47 |
+
- sanitizer: UNDEFINED
|
| 48 |
+
machine: [self-hosted, X64, Linux]
|
| 49 |
+
|
| 50 |
+
runs-on: ${{ matrix.machine }}
|
| 51 |
+
|
| 52 |
+
steps:
|
| 53 |
+
- name: Clone
|
| 54 |
+
id: checkout
|
| 55 |
+
uses: actions/checkout@v6
|
| 56 |
+
|
| 57 |
+
# - name: ccache
|
| 58 |
+
# uses: ggml-org/ccache-action@v1.2.24
|
| 59 |
+
# if: ${{ matrix.sanitizer != 'UNDEFINED' }}
|
| 60 |
+
# with:
|
| 61 |
+
# key: ctest-${{ matrix.sanitizer }}-ubuntu-24.04
|
| 62 |
+
# variant: ccache
|
| 63 |
+
# evict-old-files: 1d
|
| 64 |
+
# save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
| 65 |
+
|
| 66 |
+
# with UNDEFINED sanitizer, we have to build in Debug to avoid GCC 13 false-positive warnings
|
| 67 |
+
- name: Build (undefined)
|
| 68 |
+
id: cmake_build_undefined
|
| 69 |
+
if: ${{ matrix.sanitizer == 'UNDEFINED' }}
|
| 70 |
+
run: |
|
| 71 |
+
cmake -B build \
|
| 72 |
+
-DCMAKE_BUILD_TYPE=Debug \
|
| 73 |
+
-DLLAMA_FATAL_WARNINGS=ON \
|
| 74 |
+
-DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \
|
| 75 |
+
-DGGML_SANITIZE_${{ matrix.sanitizer }}=ON
|
| 76 |
+
|
| 77 |
+
cmake --build build --config Debug -j $(nproc)
|
| 78 |
+
|
| 79 |
+
- name: Build
|
| 80 |
+
id: cmake_build
|
| 81 |
+
if: ${{ matrix.sanitizer == 'ADDRESS' }}
|
| 82 |
+
run: |
|
| 83 |
+
cmake -B build \
|
| 84 |
+
-DCMAKE_BUILD_TYPE=RelWithDebInfo \
|
| 85 |
+
-DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \
|
| 86 |
+
-DGGML_SANITIZE_${{ matrix.sanitizer }}=ON
|
| 87 |
+
|
| 88 |
+
cmake --build build --config RelWithDebInfo -j $(nproc)
|
| 89 |
+
|
| 90 |
+
- name: Build (no OpenMP)
|
| 91 |
+
id: cmake_build_no_openmp
|
| 92 |
+
if: ${{ matrix.sanitizer == 'THREAD' }}
|
| 93 |
+
run: |
|
| 94 |
+
cmake -B build \
|
| 95 |
+
-DCMAKE_BUILD_TYPE=RelWithDebInfo \
|
| 96 |
+
-DLLAMA_SANITIZE_${{ matrix.sanitizer }}=ON \
|
| 97 |
+
-DGGML_SANITIZE_${{ matrix.sanitizer }}=ON \
|
| 98 |
+
-DGGML_OPENMP=OFF
|
| 99 |
+
|
| 100 |
+
cmake --build build --config RelWithDebInfo -j $(nproc)
|
| 101 |
+
|
| 102 |
+
- name: Test
|
| 103 |
+
id: cmake_test
|
| 104 |
+
# skip run in Debug - very slow
|
| 105 |
+
if: ${{ matrix.sanitizer != 'UNDEFINED' }}
|
| 106 |
+
run: |
|
| 107 |
+
cd build
|
| 108 |
+
ctest -L main -E tokenizer --verbose --timeout 900
|
.github/workflows/build-self-hosted.yml
ADDED
|
@@ -0,0 +1,409 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: CI (self-hosted)
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
workflow_dispatch: # allows manual triggering
|
| 5 |
+
push:
|
| 6 |
+
branches:
|
| 7 |
+
- master
|
| 8 |
+
paths: [
|
| 9 |
+
'.github/workflows/build-self-hosted.yml',
|
| 10 |
+
'**/CMakeLists.txt',
|
| 11 |
+
'**/.cmake',
|
| 12 |
+
'**/*.h',
|
| 13 |
+
'**/*.hpp',
|
| 14 |
+
'**/*.c',
|
| 15 |
+
'**/*.cpp',
|
| 16 |
+
'**/*.cu',
|
| 17 |
+
'**/*.cuh',
|
| 18 |
+
'**/*.swift',
|
| 19 |
+
'**/*.m',
|
| 20 |
+
'**/*.metal',
|
| 21 |
+
'**/*.comp',
|
| 22 |
+
'**/*.glsl',
|
| 23 |
+
'**/*.wgsl'
|
| 24 |
+
]
|
| 25 |
+
|
| 26 |
+
pull_request:
|
| 27 |
+
types: [opened, synchronize, reopened]
|
| 28 |
+
paths: [
|
| 29 |
+
'.github/workflows/build-self-hosted.yml',
|
| 30 |
+
'**/CMakeLists.txt',
|
| 31 |
+
'**/.cmake',
|
| 32 |
+
'**/*.h',
|
| 33 |
+
'**/*.hpp',
|
| 34 |
+
'**/*.c',
|
| 35 |
+
'**/*.cpp',
|
| 36 |
+
'**/*.cu',
|
| 37 |
+
'**/*.cuh',
|
| 38 |
+
'**/*.swift',
|
| 39 |
+
'**/*.m',
|
| 40 |
+
'**/*.metal',
|
| 41 |
+
'**/*.comp',
|
| 42 |
+
'**/*.glsl',
|
| 43 |
+
'**/*.wgsl'
|
| 44 |
+
]
|
| 45 |
+
|
| 46 |
+
concurrency:
|
| 47 |
+
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
| 48 |
+
cancel-in-progress: true
|
| 49 |
+
|
| 50 |
+
env:
|
| 51 |
+
# note: this is dud token to avoid rate limiting (https://github.com/ggml-org/llama.cpp/pull/25706#issuecomment-4979941302)
|
| 52 |
+
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
| 53 |
+
GGML_NLOOP: 3
|
| 54 |
+
GGML_N_THREADS: 1
|
| 55 |
+
LLAMA_ARG_LOG_COLORS: 1
|
| 56 |
+
LLAMA_ARG_LOG_PREFIX: 1
|
| 57 |
+
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
| 58 |
+
|
| 59 |
+
jobs:
|
| 60 |
+
gpu-cuda:
|
| 61 |
+
runs-on: [self-hosted, Linux, NVIDIA]
|
| 62 |
+
|
| 63 |
+
steps:
|
| 64 |
+
- name: Clone
|
| 65 |
+
id: checkout
|
| 66 |
+
uses: actions/checkout@v6
|
| 67 |
+
|
| 68 |
+
- name: Test
|
| 69 |
+
id: ggml-ci
|
| 70 |
+
run: |
|
| 71 |
+
nvidia-smi
|
| 72 |
+
GG_BUILD_CUDA=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
| 73 |
+
|
| 74 |
+
gpu-rocm:
|
| 75 |
+
runs-on: [self-hosted, Linux, AMD]
|
| 76 |
+
|
| 77 |
+
steps:
|
| 78 |
+
- name: Clone
|
| 79 |
+
id: checkout
|
| 80 |
+
uses: actions/checkout@v6
|
| 81 |
+
|
| 82 |
+
- name: Test
|
| 83 |
+
id: ggml-ci
|
| 84 |
+
# HIP_LAUNCH_BLOCKING=1: workaround for an async-execution correctness
|
| 85 |
+
# issue on integrated RDNA3.5 (gfx1151) where batched inference returns
|
| 86 |
+
# incorrect output (perplexity ~88 vs ~9.4). Serializing kernel launches
|
| 87 |
+
# restores correctness. Remove once the underlying ROCm/HIP issue is fixed.
|
| 88 |
+
env:
|
| 89 |
+
HIP_LAUNCH_BLOCKING: "1"
|
| 90 |
+
run: |
|
| 91 |
+
rocminfo
|
| 92 |
+
GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS=gfx1151 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
| 93 |
+
|
| 94 |
+
gpu-vulkan-nvidia-cm:
|
| 95 |
+
runs-on: [self-hosted, Linux, NVIDIA]
|
| 96 |
+
|
| 97 |
+
steps:
|
| 98 |
+
- name: Clone
|
| 99 |
+
id: checkout
|
| 100 |
+
uses: actions/checkout@v6
|
| 101 |
+
|
| 102 |
+
- name: Test
|
| 103 |
+
id: ggml-ci
|
| 104 |
+
run: |
|
| 105 |
+
vulkaninfo --summary
|
| 106 |
+
GG_BUILD_VULKAN=1 GGML_VK_DISABLE_COOPMAT2=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
| 107 |
+
|
| 108 |
+
gpu-vulkan-nvidia-cm2:
|
| 109 |
+
runs-on: [self-hosted, Linux, NVIDIA, COOPMAT2]
|
| 110 |
+
|
| 111 |
+
steps:
|
| 112 |
+
- name: Clone
|
| 113 |
+
id: checkout
|
| 114 |
+
uses: actions/checkout@v6
|
| 115 |
+
|
| 116 |
+
- name: Test
|
| 117 |
+
id: ggml-ci
|
| 118 |
+
run: |
|
| 119 |
+
vulkaninfo --summary
|
| 120 |
+
GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
| 121 |
+
|
| 122 |
+
gpu-webgpu-nvidia:
|
| 123 |
+
runs-on: [self-hosted, Linux, NVIDIA, X64]
|
| 124 |
+
|
| 125 |
+
steps:
|
| 126 |
+
- name: Clone
|
| 127 |
+
id: checkout
|
| 128 |
+
uses: actions/checkout@v6
|
| 129 |
+
|
| 130 |
+
- name: Dawn Dependency
|
| 131 |
+
id: dawn-depends
|
| 132 |
+
run: |
|
| 133 |
+
DAWN_VERSION="v20260317.182325"
|
| 134 |
+
DAWN_OWNER="google"
|
| 135 |
+
DAWN_REPO="dawn"
|
| 136 |
+
DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-ubuntu-latest-Release"
|
| 137 |
+
echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
|
| 138 |
+
curl -L -o artifact.tar.gz \
|
| 139 |
+
"https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
|
| 140 |
+
mkdir dawn
|
| 141 |
+
tar -xvf artifact.tar.gz -C dawn --strip-components=1
|
| 142 |
+
|
| 143 |
+
- name: Test
|
| 144 |
+
id: ggml-ci
|
| 145 |
+
run: |
|
| 146 |
+
GG_BUILD_WEBGPU=1 \
|
| 147 |
+
GG_BUILD_WEBGPU_DAWN_PREFIX="$GITHUB_WORKSPACE/dawn" \
|
| 148 |
+
GG_BUILD_WEBGPU_DAWN_DIR="$GITHUB_WORKSPACE/dawn/lib64/cmake/Dawn" \
|
| 149 |
+
bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
| 150 |
+
|
| 151 |
+
# TODO: provision AMX-compatible machine
|
| 152 |
+
#cpu-amx:
|
| 153 |
+
# runs-on: [self-hosted, Linux, CPU, AMX]
|
| 154 |
+
|
| 155 |
+
# steps:
|
| 156 |
+
# - name: Clone
|
| 157 |
+
# id: checkout
|
| 158 |
+
# uses: actions/checkout@v6
|
| 159 |
+
|
| 160 |
+
# - name: Test
|
| 161 |
+
# id: ggml-ci
|
| 162 |
+
# run: |
|
| 163 |
+
# bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
| 164 |
+
|
| 165 |
+
# TODO: provision AMD GPU machine
|
| 166 |
+
# amd-vulkan:
|
| 167 |
+
# runs-on: [self-hosted, Linux, AMD]
|
| 168 |
+
|
| 169 |
+
# steps:
|
| 170 |
+
# - name: Clone
|
| 171 |
+
# id: checkout
|
| 172 |
+
# uses: actions/checkout@v6
|
| 173 |
+
|
| 174 |
+
# - name: Test
|
| 175 |
+
# id: ggml-ci
|
| 176 |
+
# run: |
|
| 177 |
+
# vulkaninfo --summary
|
| 178 |
+
# GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
| 179 |
+
|
| 180 |
+
# TODO: provision AMD GPU machine
|
| 181 |
+
# amd-rocm:
|
| 182 |
+
# runs-on: [self-hosted, Linux, AMD]
|
| 183 |
+
|
| 184 |
+
# steps:
|
| 185 |
+
# - name: Clone
|
| 186 |
+
# id: checkout
|
| 187 |
+
# uses: actions/checkout@v6
|
| 188 |
+
|
| 189 |
+
# - name: Test
|
| 190 |
+
# id: ggml-ci
|
| 191 |
+
# run: |
|
| 192 |
+
# amd-smi static
|
| 193 |
+
# GG_BUILD_ROCM=1 GG_BUILD_AMDGPU_TARGETS="gfx1101" bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
| 194 |
+
|
| 195 |
+
gpu-metal:
|
| 196 |
+
runs-on: [self-hosted, macOS, ARM64]
|
| 197 |
+
|
| 198 |
+
steps:
|
| 199 |
+
- name: Clone
|
| 200 |
+
id: checkout
|
| 201 |
+
uses: actions/checkout@v6
|
| 202 |
+
|
| 203 |
+
- name: Test
|
| 204 |
+
id: ggml-ci
|
| 205 |
+
run: |
|
| 206 |
+
GG_BUILD_METAL=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
| 207 |
+
|
| 208 |
+
gpu-webgpu-apple:
|
| 209 |
+
runs-on: [self-hosted, macOS, ARM64]
|
| 210 |
+
|
| 211 |
+
steps:
|
| 212 |
+
- name: Clone
|
| 213 |
+
id: checkout
|
| 214 |
+
uses: actions/checkout@v6
|
| 215 |
+
|
| 216 |
+
- name: Dawn Dependency
|
| 217 |
+
id: dawn-depends
|
| 218 |
+
run: |
|
| 219 |
+
DAWN_VERSION="v20260317.182325"
|
| 220 |
+
DAWN_OWNER="google"
|
| 221 |
+
DAWN_REPO="dawn"
|
| 222 |
+
DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-macos-latest-Release"
|
| 223 |
+
echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
|
| 224 |
+
curl -L -o artifact.tar.gz \
|
| 225 |
+
"https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
|
| 226 |
+
mkdir dawn
|
| 227 |
+
tar -xvf artifact.tar.gz -C dawn --strip-components=1
|
| 228 |
+
|
| 229 |
+
- name: Test
|
| 230 |
+
id: ggml-ci
|
| 231 |
+
run: |
|
| 232 |
+
GG_BUILD_WEBGPU=1 GG_BUILD_WEBGPU_DAWN_PREFIX="$GITHUB_WORKSPACE/dawn" \
|
| 233 |
+
bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
| 234 |
+
|
| 235 |
+
gpu-vulkan-apple:
|
| 236 |
+
runs-on: [self-hosted, macOS, ARM64]
|
| 237 |
+
|
| 238 |
+
steps:
|
| 239 |
+
- name: Clone
|
| 240 |
+
id: checkout
|
| 241 |
+
uses: actions/checkout@v6
|
| 242 |
+
|
| 243 |
+
- name: Test
|
| 244 |
+
id: ggml-ci
|
| 245 |
+
run: |
|
| 246 |
+
vulkaninfo --summary
|
| 247 |
+
GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
| 248 |
+
|
| 249 |
+
gpu-vulkan-intel-linux:
|
| 250 |
+
runs-on: [self-hosted, Linux, Intel]
|
| 251 |
+
|
| 252 |
+
steps:
|
| 253 |
+
- name: Clone
|
| 254 |
+
id: checkout
|
| 255 |
+
uses: actions/checkout@v6
|
| 256 |
+
with:
|
| 257 |
+
persist-credentials: false
|
| 258 |
+
|
| 259 |
+
- name: Test
|
| 260 |
+
id: ggml-ci
|
| 261 |
+
run: |
|
| 262 |
+
vulkaninfo --summary
|
| 263 |
+
GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
| 264 |
+
|
| 265 |
+
gpu-vulkan-intel-windows:
|
| 266 |
+
runs-on: [self-hosted, Windows, X64, Intel]
|
| 267 |
+
|
| 268 |
+
steps:
|
| 269 |
+
- name: Clone
|
| 270 |
+
id: checkout
|
| 271 |
+
uses: actions/checkout@v6
|
| 272 |
+
|
| 273 |
+
- name: Test
|
| 274 |
+
id: ggml-ci
|
| 275 |
+
shell: C:\msys64\usr\bin\bash.exe --noprofile --norc -eo pipefail "{0}"
|
| 276 |
+
env:
|
| 277 |
+
MSYSTEM: UCRT64
|
| 278 |
+
CHERE_INVOKING: 1
|
| 279 |
+
PATH: C:\msys64\ucrt64\bin;C:\msys64\usr\bin;C:\Windows\System32;${{ env.PATH }}
|
| 280 |
+
run: |
|
| 281 |
+
vulkaninfo --summary
|
| 282 |
+
# Skip python related tests with GG_BUILD_LOW_PERF=1 since Windows MSYS2 UCRT64 currently fails to create
|
| 283 |
+
# a valid python environment for testing
|
| 284 |
+
LLAMA_FATAL_WARNINGS=OFF GG_BUILD_NINJA=1 GG_BUILD_VULKAN=1 GG_BUILD_LOW_PERF=1 ./ci/run.sh ./results/llama.cpp ./mnt/llama.cpp
|
| 285 |
+
|
| 286 |
+
gpu-openvino-low-perf:
|
| 287 |
+
runs-on: [self-hosted, Linux, Intel, OpenVINO]
|
| 288 |
+
|
| 289 |
+
env:
|
| 290 |
+
# Sync versions in build.yml, build-self-hosted.yml, release.yml, build-cache.yml, .devops/openvino.Dockerfile
|
| 291 |
+
OPENVINO_VERSION_MAJOR: "2026.3.1"
|
| 292 |
+
OPENVINO_VERSION_FULL: "2026.3.1.22476.56d9685302d"
|
| 293 |
+
|
| 294 |
+
steps:
|
| 295 |
+
- name: Clone
|
| 296 |
+
id: checkout
|
| 297 |
+
uses: actions/checkout@v6
|
| 298 |
+
|
| 299 |
+
- name: Setup OpenVINO Toolkit
|
| 300 |
+
uses: ./.github/actions/linux-setup-openvino
|
| 301 |
+
with:
|
| 302 |
+
path: ./openvino_toolkit
|
| 303 |
+
version_major: ${{ env.OPENVINO_VERSION_MAJOR }}
|
| 304 |
+
version_full: ${{ env.OPENVINO_VERSION_FULL }}
|
| 305 |
+
|
| 306 |
+
- name: Install OpenVINO dependencies
|
| 307 |
+
run: |
|
| 308 |
+
cd ./openvino_toolkit
|
| 309 |
+
chmod +x ./install_dependencies/install_openvino_dependencies.sh
|
| 310 |
+
echo "Y" | sudo -E ./install_dependencies/install_openvino_dependencies.sh
|
| 311 |
+
|
| 312 |
+
- name: Test
|
| 313 |
+
id: ggml-ci
|
| 314 |
+
run: |
|
| 315 |
+
source ./openvino_toolkit/setupvars.sh
|
| 316 |
+
GG_BUILD_OPENVINO=1 GGML_OPENVINO_DEVICE=GPU GG_BUILD_LOW_PERF=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
| 317 |
+
|
| 318 |
+
cpu-x64-high-perf:
|
| 319 |
+
runs-on: [self-hosted, Linux, X64]
|
| 320 |
+
|
| 321 |
+
steps:
|
| 322 |
+
- name: Clone
|
| 323 |
+
id: checkout
|
| 324 |
+
uses: actions/checkout@v6
|
| 325 |
+
|
| 326 |
+
- name: Test
|
| 327 |
+
id: ggml-ci
|
| 328 |
+
run: |
|
| 329 |
+
LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
| 330 |
+
|
| 331 |
+
cpu-arm64-high-perf-graviton4:
|
| 332 |
+
runs-on: ah-ubuntu_22_04-c8g_8x
|
| 333 |
+
|
| 334 |
+
steps:
|
| 335 |
+
- name: Clone
|
| 336 |
+
id: checkout
|
| 337 |
+
uses: actions/checkout@v6
|
| 338 |
+
|
| 339 |
+
- name: Dependencies
|
| 340 |
+
id: depends
|
| 341 |
+
run: |
|
| 342 |
+
set -euxo pipefail
|
| 343 |
+
sudo apt-get update
|
| 344 |
+
sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a \
|
| 345 |
+
apt-get install -y \
|
| 346 |
+
build-essential \
|
| 347 |
+
python3-venv \
|
| 348 |
+
gpg \
|
| 349 |
+
wget \
|
| 350 |
+
time \
|
| 351 |
+
git-lfs
|
| 352 |
+
|
| 353 |
+
git lfs install
|
| 354 |
+
|
| 355 |
+
# install the latest cmake
|
| 356 |
+
sudo install -d /usr/share/keyrings
|
| 357 |
+
wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc \
|
| 358 |
+
| gpg --dearmor \
|
| 359 |
+
| sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
|
| 360 |
+
echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ jammy main' \
|
| 361 |
+
| sudo tee /etc/apt/sources.list.d/kitware.list
|
| 362 |
+
sudo apt-get update
|
| 363 |
+
sudo apt-get install -y cmake
|
| 364 |
+
|
| 365 |
+
- name: Test
|
| 366 |
+
id: ggml-ci
|
| 367 |
+
run: |
|
| 368 |
+
LLAMA_ARG_THREADS=$(nproc) GG_BUILD_HIGH_PERF=1 GG_BUILD_NO_BF16=1 GG_BUILD_EXTRA_TESTS_0=1 bash ./ci/run.sh ~/results/llama.cpp ~/mnt/llama.cpp
|
| 369 |
+
|
| 370 |
+
cpu-arm64-graviton4-kleidiai:
|
| 371 |
+
runs-on: ah-ubuntu_22_04-c8g_8x
|
| 372 |
+
|
| 373 |
+
steps:
|
| 374 |
+
- name: Clone
|
| 375 |
+
id: checkout
|
| 376 |
+
uses: actions/checkout@v6
|
| 377 |
+
|
| 378 |
+
- name: Dependencies
|
| 379 |
+
id: depends
|
| 380 |
+
run: |
|
| 381 |
+
set -euxo pipefail
|
| 382 |
+
sudo apt-get update
|
| 383 |
+
sudo DEBIAN_FRONTEND=noninteractive NEEDRESTART_MODE=a \
|
| 384 |
+
apt-get install -y \
|
| 385 |
+
build-essential \
|
| 386 |
+
python3-venv \
|
| 387 |
+
gpg \
|
| 388 |
+
wget \
|
| 389 |
+
time \
|
| 390 |
+
git-lfs
|
| 391 |
+
|
| 392 |
+
git lfs install
|
| 393 |
+
|
| 394 |
+
# install the latest cmake
|
| 395 |
+
sudo install -d /usr/share/keyrings
|
| 396 |
+
wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc \
|
| 397 |
+
| gpg --dearmor \
|
| 398 |
+
| sudo tee /usr/share/keyrings/kitware-archive-keyring.gpg >/dev/null
|
| 399 |
+
echo 'deb [signed-by=/usr/share/keyrings/kitware-archive-keyring.gpg] https://apt.kitware.com/ubuntu/ jammy main' \
|
| 400 |
+
| sudo tee /etc/apt/sources.list.d/kitware.list
|
| 401 |
+
sudo apt-get update
|
| 402 |
+
sudo apt-get install -y cmake
|
| 403 |
+
|
| 404 |
+
- name: Test
|
| 405 |
+
id: ggml-ci
|
| 406 |
+
run: |
|
| 407 |
+
GG_BUILD_KLEIDIAI=1 \
|
| 408 |
+
GG_BUILD_EXTRA_TESTS_0=1 \
|
| 409 |
+
bash ./ci/run.sh ./tmp/results ./tmp/mnt
|
.github/workflows/build-sycl.yml
ADDED
|
@@ -0,0 +1,171 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: CI (sycl)
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
workflow_dispatch: # allows manual triggering
|
| 5 |
+
push:
|
| 6 |
+
branches:
|
| 7 |
+
- master
|
| 8 |
+
paths: [
|
| 9 |
+
'.github/workflows/build-sycl.yml',
|
| 10 |
+
'**/CMakeLists.txt',
|
| 11 |
+
'**/.cmake',
|
| 12 |
+
'**/*.h',
|
| 13 |
+
'**/*.hpp',
|
| 14 |
+
'**/*.c',
|
| 15 |
+
'**/*.cpp'
|
| 16 |
+
]
|
| 17 |
+
|
| 18 |
+
pull_request:
|
| 19 |
+
types: [opened, synchronize, reopened]
|
| 20 |
+
paths: [
|
| 21 |
+
'.github/workflows/build-sycl.yml',
|
| 22 |
+
'ggml/src/ggml-sycl/**'
|
| 23 |
+
]
|
| 24 |
+
|
| 25 |
+
concurrency:
|
| 26 |
+
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
| 27 |
+
cancel-in-progress: true
|
| 28 |
+
|
| 29 |
+
env:
|
| 30 |
+
GGML_NLOOP: 3
|
| 31 |
+
GGML_N_THREADS: 1
|
| 32 |
+
LLAMA_ARG_LOG_COLORS: 1
|
| 33 |
+
LLAMA_ARG_LOG_PREFIX: 1
|
| 34 |
+
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
| 35 |
+
|
| 36 |
+
jobs:
|
| 37 |
+
ubuntu-24-sycl:
|
| 38 |
+
strategy:
|
| 39 |
+
matrix:
|
| 40 |
+
build: [fp32, fp16]
|
| 41 |
+
include:
|
| 42 |
+
- build: fp32
|
| 43 |
+
fp16: OFF
|
| 44 |
+
- build: fp16
|
| 45 |
+
fp16: ON
|
| 46 |
+
|
| 47 |
+
runs-on: ubuntu-24.04
|
| 48 |
+
|
| 49 |
+
env:
|
| 50 |
+
ONEAPI_ROOT: /opt/intel/oneapi/
|
| 51 |
+
ONEAPI_INSTALLER_VERSION: "2025.3.3"
|
| 52 |
+
LEVEL_ZERO_VERSION: "1.28.2"
|
| 53 |
+
LEVEL_ZERO_UBUNTU_VERSION: "u24.04"
|
| 54 |
+
|
| 55 |
+
continue-on-error: true
|
| 56 |
+
|
| 57 |
+
steps:
|
| 58 |
+
- name: Clone
|
| 59 |
+
id: checkout
|
| 60 |
+
uses: actions/checkout@v6
|
| 61 |
+
|
| 62 |
+
- name: Download & Install oneAPI
|
| 63 |
+
shell: bash
|
| 64 |
+
run: |
|
| 65 |
+
cd /tmp
|
| 66 |
+
wget https://registrationcenter-download.intel.com/akdlm/IRC_NAS/56f7923a-adb8-43f3-8b02-2b60fcac8cab/intel-deep-learning-essentials-2025.3.3.16_offline.sh -O intel-deep-learning-essentials_offline.sh
|
| 67 |
+
sudo bash intel-deep-learning-essentials_offline.sh -s -a --silent --eula accept
|
| 68 |
+
|
| 69 |
+
- name: Install Level Zero SDK
|
| 70 |
+
shell: bash
|
| 71 |
+
run: |
|
| 72 |
+
cd /tmp
|
| 73 |
+
wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero.deb
|
| 74 |
+
wget -q "https://github.com/oneapi-src/level-zero/releases/download/v${LEVEL_ZERO_VERSION}/level-zero-devel_${LEVEL_ZERO_VERSION}%2B${LEVEL_ZERO_UBUNTU_VERSION}_amd64.deb" -O level-zero-devel.deb
|
| 75 |
+
sudo apt-get install -y ./level-zero.deb ./level-zero-devel.deb
|
| 76 |
+
|
| 77 |
+
- name: ccache
|
| 78 |
+
uses: ggml-org/ccache-action@v1.2.24
|
| 79 |
+
with:
|
| 80 |
+
key: sycl-ubuntu-24-${{ matrix.build }}
|
| 81 |
+
save: false
|
| 82 |
+
|
| 83 |
+
- name: ccache-buckets-restore
|
| 84 |
+
uses: ./.github/actions/ccache-buckets
|
| 85 |
+
env:
|
| 86 |
+
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
| 87 |
+
with:
|
| 88 |
+
key: sycl-ubuntu-24-${{ matrix.build }}
|
| 89 |
+
folder: llama.cpp
|
| 90 |
+
hf_bucket: ggml-org/cache
|
| 91 |
+
|
| 92 |
+
- name: Build
|
| 93 |
+
id: cmake_build
|
| 94 |
+
run: |
|
| 95 |
+
source /opt/intel/oneapi/setvars.sh
|
| 96 |
+
cmake -B build \
|
| 97 |
+
-G "Ninja" \
|
| 98 |
+
-DCMAKE_BUILD_TYPE=Release \
|
| 99 |
+
-DGGML_SYCL=ON \
|
| 100 |
+
-DCMAKE_C_COMPILER=icx \
|
| 101 |
+
-DCMAKE_CXX_COMPILER=icpx \
|
| 102 |
+
-DLLAMA_OPENSSL=OFF \
|
| 103 |
+
-DGGML_NATIVE=OFF \
|
| 104 |
+
-DGGML_SYCL_F16=${{ matrix.fp16 }}
|
| 105 |
+
time cmake --build build --config Release -j $(nproc)
|
| 106 |
+
|
| 107 |
+
- name: ccache-buckets-save
|
| 108 |
+
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
| 109 |
+
uses: ./.github/actions/ccache-buckets
|
| 110 |
+
env:
|
| 111 |
+
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
| 112 |
+
with:
|
| 113 |
+
key: sycl-ubuntu-24-${{ matrix.build }}
|
| 114 |
+
folder: llama.cpp
|
| 115 |
+
evict-old-files: 1d
|
| 116 |
+
hf_bucket: ggml-org/cache
|
| 117 |
+
save: true
|
| 118 |
+
|
| 119 |
+
windows-latest-sycl:
|
| 120 |
+
runs-on: windows-2022
|
| 121 |
+
|
| 122 |
+
defaults:
|
| 123 |
+
run:
|
| 124 |
+
shell: bash
|
| 125 |
+
|
| 126 |
+
env:
|
| 127 |
+
WINDOWS_BASEKIT_URL: https://registrationcenter-download.intel.com/akdlm/IRC_NAS/b60765d1-2b85-4e85-86b6-cb0e9563a699/intel-deep-learning-essentials-2025.3.3.18_offline.exe
|
| 128 |
+
WINDOWS_DPCPP_MKL: intel.oneapi.win.cpp-dpcpp-common:intel.oneapi.win.mkl.devel:intel.oneapi.win.dnnl:intel.oneapi.win.tbb.devel
|
| 129 |
+
LEVEL_ZERO_SDK_URL: https://github.com/oneapi-src/level-zero/releases/download/v1.28.2/level-zero-win-sdk-1.28.2.zip
|
| 130 |
+
ONEAPI_ROOT: "C:/Program Files (x86)/Intel/oneAPI"
|
| 131 |
+
ONEAPI_INSTALLER_VERSION: "2025.3.3"
|
| 132 |
+
steps:
|
| 133 |
+
- name: Clone
|
| 134 |
+
id: checkout
|
| 135 |
+
uses: actions/checkout@v6
|
| 136 |
+
|
| 137 |
+
- name: Download & Install oneAPI
|
| 138 |
+
shell: bash
|
| 139 |
+
run: |
|
| 140 |
+
scripts/install-oneapi.bat $WINDOWS_BASEKIT_URL $WINDOWS_DPCPP_MKL
|
| 141 |
+
|
| 142 |
+
- name: Install Level Zero SDK
|
| 143 |
+
shell: pwsh
|
| 144 |
+
run: |
|
| 145 |
+
Invoke-WebRequest -Uri "${{ env.LEVEL_ZERO_SDK_URL }}" -OutFile "level-zero-win-sdk.zip"
|
| 146 |
+
Expand-Archive -Path "level-zero-win-sdk.zip" -DestinationPath "C:/level-zero-sdk" -Force
|
| 147 |
+
"LEVEL_ZERO_V1_SDK_PATH=C:/level-zero-sdk" | Out-File -FilePath $env:GITHUB_ENV -Append
|
| 148 |
+
|
| 149 |
+
- name: ccache
|
| 150 |
+
uses: ggml-org/ccache-action@v1.2.24
|
| 151 |
+
with:
|
| 152 |
+
key: sycl-windows-latest
|
| 153 |
+
variant: ccache
|
| 154 |
+
evict-old-files: 1d
|
| 155 |
+
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
| 156 |
+
|
| 157 |
+
# TODO: add ssl support ; we will also need to modify win-build-sycl.bat to accept user-specified args
|
| 158 |
+
|
| 159 |
+
- name: Build
|
| 160 |
+
id: cmake_build
|
| 161 |
+
run: examples/sycl/win-build-sycl.bat
|
| 162 |
+
|
| 163 |
+
- name: ccache-clear
|
| 164 |
+
uses: ./.github/actions/ccache-clear
|
| 165 |
+
env:
|
| 166 |
+
GH_TOKEN: ${{ github.token }}
|
| 167 |
+
with:
|
| 168 |
+
key: sycl-windows-latest
|
| 169 |
+
older: 5m
|
| 170 |
+
min: 1
|
| 171 |
+
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
|
.github/workflows/build-virtgpu.yml
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: CI (virtgpu)
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
workflow_dispatch: # allows manual triggering
|
| 5 |
+
push:
|
| 6 |
+
branches:
|
| 7 |
+
- master
|
| 8 |
+
paths: [
|
| 9 |
+
'.github/workflows/build-virtgpu.yml',
|
| 10 |
+
'**/CMakeLists.txt',
|
| 11 |
+
'**/.cmake',
|
| 12 |
+
'**/*.h',
|
| 13 |
+
'**/*.hpp',
|
| 14 |
+
'**/*.c',
|
| 15 |
+
'**/*.cpp'
|
| 16 |
+
]
|
| 17 |
+
|
| 18 |
+
pull_request:
|
| 19 |
+
types: [opened, synchronize, reopened]
|
| 20 |
+
paths: [
|
| 21 |
+
'.github/workflows/build-virtgpu.yml',
|
| 22 |
+
'ggml/src/ggml-virtgpu/**'
|
| 23 |
+
]
|
| 24 |
+
|
| 25 |
+
concurrency:
|
| 26 |
+
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
| 27 |
+
cancel-in-progress: true
|
| 28 |
+
|
| 29 |
+
jobs:
|
| 30 |
+
ubuntu-24-virtgpu:
|
| 31 |
+
runs-on: ${{ 'ubuntu-24.04-arm' || 'ubuntu-24.04' }}
|
| 32 |
+
|
| 33 |
+
steps:
|
| 34 |
+
- name: Clone
|
| 35 |
+
id: checkout
|
| 36 |
+
uses: actions/checkout@v6
|
| 37 |
+
|
| 38 |
+
- name: Dependencies
|
| 39 |
+
id: depends
|
| 40 |
+
run: |
|
| 41 |
+
sudo apt-get update
|
| 42 |
+
sudo apt-get install -y build-essential libdrm-dev pkg-config libssl-dev
|
| 43 |
+
|
| 44 |
+
- name: Build
|
| 45 |
+
id: cmake_build
|
| 46 |
+
run: |
|
| 47 |
+
cmake -B build \
|
| 48 |
+
-DGGML_VIRTGPU=ON \
|
| 49 |
+
-DGGML_VIRTGPU_BACKEND=ON
|
| 50 |
+
cmake --build build --config Release -j $(nproc)
|
.github/workflows/build-vulkan.yml
ADDED
|
@@ -0,0 +1,232 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: CI (vulkan)
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
workflow_dispatch: # allows manual triggering
|
| 5 |
+
push:
|
| 6 |
+
branches:
|
| 7 |
+
- master
|
| 8 |
+
paths: [
|
| 9 |
+
'.github/workflows/build-vulkan.yml',
|
| 10 |
+
'**/CMakeLists.txt',
|
| 11 |
+
'**/.cmake',
|
| 12 |
+
'**/*.h',
|
| 13 |
+
'**/*.hpp',
|
| 14 |
+
'**/*.c',
|
| 15 |
+
'**/*.cpp',
|
| 16 |
+
'**/*.comp',
|
| 17 |
+
'**/*.glsl'
|
| 18 |
+
]
|
| 19 |
+
|
| 20 |
+
pull_request:
|
| 21 |
+
types: [opened, synchronize, reopened]
|
| 22 |
+
paths: [
|
| 23 |
+
'.github/workflows/build-vulkan.yml',
|
| 24 |
+
'ggml/src/ggml-vulkan/**'
|
| 25 |
+
]
|
| 26 |
+
|
| 27 |
+
concurrency:
|
| 28 |
+
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
| 29 |
+
cancel-in-progress: true
|
| 30 |
+
|
| 31 |
+
env:
|
| 32 |
+
GGML_NLOOP: 3
|
| 33 |
+
GGML_N_THREADS: 1
|
| 34 |
+
LLAMA_ARG_LOG_COLORS: 1
|
| 35 |
+
LLAMA_ARG_LOG_PREFIX: 1
|
| 36 |
+
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
| 37 |
+
|
| 38 |
+
jobs:
|
| 39 |
+
ubuntu-arm64:
|
| 40 |
+
runs-on: ubuntu-24.04-arm
|
| 41 |
+
|
| 42 |
+
steps:
|
| 43 |
+
- name: Clone
|
| 44 |
+
id: checkout
|
| 45 |
+
uses: actions/checkout@v6
|
| 46 |
+
|
| 47 |
+
- name: Dependencies
|
| 48 |
+
id: depends
|
| 49 |
+
run: |
|
| 50 |
+
sudo apt-get update
|
| 51 |
+
sudo apt-get install -y gcc-14 g++-14 build-essential glslc libvulkan-dev spirv-headers libssl-dev ninja-build
|
| 52 |
+
echo "CC=gcc-14" >> "$GITHUB_ENV"
|
| 53 |
+
echo "CXX=g++-14" >> "$GITHUB_ENV"
|
| 54 |
+
|
| 55 |
+
- name: ccache
|
| 56 |
+
uses: ggml-org/ccache-action@v1.2.24
|
| 57 |
+
with:
|
| 58 |
+
key: vulkan-ubuntu-24.04-arm
|
| 59 |
+
variant: ccache
|
| 60 |
+
save: false
|
| 61 |
+
|
| 62 |
+
- name: ccache-buckets-restore
|
| 63 |
+
uses: ./.github/actions/ccache-buckets
|
| 64 |
+
env:
|
| 65 |
+
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
| 66 |
+
with:
|
| 67 |
+
key: vulkan-ubuntu-24.04-arm
|
| 68 |
+
folder: llama.cpp
|
| 69 |
+
hf_bucket: ggml-org/cache
|
| 70 |
+
|
| 71 |
+
- name: Configure
|
| 72 |
+
id: cmake_configure
|
| 73 |
+
run: |
|
| 74 |
+
cmake -B build \
|
| 75 |
+
-G "Ninja" \
|
| 76 |
+
-DCMAKE_BUILD_TYPE=Release \
|
| 77 |
+
-DGGML_VULKAN=ON
|
| 78 |
+
|
| 79 |
+
- name: Build
|
| 80 |
+
id: cmake_build
|
| 81 |
+
run: |
|
| 82 |
+
time cmake --build build -j $(nproc)
|
| 83 |
+
|
| 84 |
+
- name: ccache-buckets-save
|
| 85 |
+
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
| 86 |
+
uses: ./.github/actions/ccache-buckets
|
| 87 |
+
env:
|
| 88 |
+
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
| 89 |
+
with:
|
| 90 |
+
key: vulkan-ubuntu-24.04-arm
|
| 91 |
+
folder: llama.cpp
|
| 92 |
+
evict-old-files: 1d
|
| 93 |
+
hf_bucket: ggml-org/cache
|
| 94 |
+
save: true
|
| 95 |
+
|
| 96 |
+
ubuntu-llvmpipe:
|
| 97 |
+
runs-on: ubuntu-24.04
|
| 98 |
+
|
| 99 |
+
steps:
|
| 100 |
+
- name: Clone
|
| 101 |
+
id: checkout
|
| 102 |
+
uses: actions/checkout@v6
|
| 103 |
+
|
| 104 |
+
- name: Dependencies
|
| 105 |
+
id: depends
|
| 106 |
+
run: |
|
| 107 |
+
sudo add-apt-repository -y ppa:kisak/kisak-mesa
|
| 108 |
+
sudo apt-get update -y
|
| 109 |
+
sudo apt-get install -y build-essential mesa-vulkan-drivers libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libssl-dev
|
| 110 |
+
|
| 111 |
+
- name: Get latest Vulkan SDK version
|
| 112 |
+
id: vulkan_sdk_version
|
| 113 |
+
run: |
|
| 114 |
+
echo "VULKAN_SDK_VERSION=$(curl https://vulkan.lunarg.com/sdk/latest/linux.txt)" >> "$GITHUB_ENV"
|
| 115 |
+
|
| 116 |
+
- name: Setup Vulkan SDK
|
| 117 |
+
id: setup
|
| 118 |
+
uses: ./.github/actions/unarchive-tar
|
| 119 |
+
with:
|
| 120 |
+
url: https://sdk.lunarg.com/sdk/download/${{ env.VULKAN_SDK_VERSION }}/linux/vulkan_sdk.tar.xz
|
| 121 |
+
path: ./vulkan_sdk
|
| 122 |
+
strip: 1
|
| 123 |
+
|
| 124 |
+
- name: ccache
|
| 125 |
+
uses: ggml-org/ccache-action@v1.2.24
|
| 126 |
+
with:
|
| 127 |
+
key: vulkan-ubuntu-24.04-llvmpipe
|
| 128 |
+
save: false
|
| 129 |
+
|
| 130 |
+
- name: ccache-buckets-restore
|
| 131 |
+
uses: ./.github/actions/ccache-buckets
|
| 132 |
+
env:
|
| 133 |
+
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
| 134 |
+
with:
|
| 135 |
+
key: vulkan-ubuntu-24.04-llvmpipe
|
| 136 |
+
folder: llama.cpp
|
| 137 |
+
hf_bucket: ggml-org/cache
|
| 138 |
+
|
| 139 |
+
- name: Build
|
| 140 |
+
id: cmake_build
|
| 141 |
+
run: |
|
| 142 |
+
source ./vulkan_sdk/setup-env.sh
|
| 143 |
+
cmake -B build \
|
| 144 |
+
-DGGML_NATIVE=OFF \
|
| 145 |
+
-DGGML_VULKAN=ON
|
| 146 |
+
cmake --build build --config Release -j $(nproc)
|
| 147 |
+
|
| 148 |
+
- name: ccache-buckets-save
|
| 149 |
+
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
| 150 |
+
uses: ./.github/actions/ccache-buckets
|
| 151 |
+
env:
|
| 152 |
+
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
| 153 |
+
with:
|
| 154 |
+
key: vulkan-ubuntu-24.04-llvmpipe
|
| 155 |
+
folder: llama.cpp
|
| 156 |
+
evict-old-files: 1d
|
| 157 |
+
hf_bucket: ggml-org/cache
|
| 158 |
+
save: true
|
| 159 |
+
|
| 160 |
+
- name: Test
|
| 161 |
+
id: cmake_test
|
| 162 |
+
run: |
|
| 163 |
+
cd build
|
| 164 |
+
export GGML_VK_VISIBLE_DEVICES=0
|
| 165 |
+
export GGML_VK_DISABLE_F16=1
|
| 166 |
+
export GGML_VK_DISABLE_COOPMAT=1
|
| 167 |
+
# This is using llvmpipe and runs slower than other backends
|
| 168 |
+
# test-backend-ops is too slow on llvmpipe, skip it
|
| 169 |
+
ctest -L main -E test-backend-ops --verbose --timeout 900
|
| 170 |
+
|
| 171 |
+
windows:
|
| 172 |
+
runs-on: windows-2025
|
| 173 |
+
|
| 174 |
+
env:
|
| 175 |
+
VULKAN_VERSION: 1.4.357.0
|
| 176 |
+
|
| 177 |
+
steps:
|
| 178 |
+
- name: Clone
|
| 179 |
+
id: checkout
|
| 180 |
+
uses: actions/checkout@v6
|
| 181 |
+
|
| 182 |
+
- name: ccache
|
| 183 |
+
uses: ggml-org/ccache-action@v1.2.24
|
| 184 |
+
with:
|
| 185 |
+
key: cpu-windows-2025-x64-vulkan
|
| 186 |
+
variant: ccache
|
| 187 |
+
evict-old-files: 1d
|
| 188 |
+
save: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
| 189 |
+
|
| 190 |
+
- name: Install Vulkan SDK
|
| 191 |
+
id: get_vulkan
|
| 192 |
+
run: |
|
| 193 |
+
curl.exe -o $env:RUNNER_TEMP/VulkanSDK-Installer.exe -L "https://sdk.lunarg.com/sdk/download/${env:VULKAN_VERSION}/windows/vulkansdk-windows-X64-${env:VULKAN_VERSION}.exe"
|
| 194 |
+
& "$env:RUNNER_TEMP\VulkanSDK-Installer.exe" --accept-licenses --default-answer --confirm-command install
|
| 195 |
+
Add-Content $env:GITHUB_ENV "VULKAN_SDK=C:\VulkanSDK\${env:VULKAN_VERSION}"
|
| 196 |
+
Add-Content $env:GITHUB_PATH "C:\VulkanSDK\${env:VULKAN_VERSION}\bin"
|
| 197 |
+
|
| 198 |
+
- name: Install Ninja
|
| 199 |
+
id: install_ninja
|
| 200 |
+
run: |
|
| 201 |
+
choco install ninja
|
| 202 |
+
|
| 203 |
+
- name: Build
|
| 204 |
+
id: cmake_build
|
| 205 |
+
run: |
|
| 206 |
+
cmake -S . -B build -G "Ninja Multi-Config" `
|
| 207 |
+
-D CMAKE_TOOLCHAIN_FILE=cmake/x64-windows-llvm.cmake `
|
| 208 |
+
-DCMAKE_BUILD_TYPE=Release `
|
| 209 |
+
-DGGML_NATIVE=OFF `
|
| 210 |
+
-DLLAMA_BUILD_SERVER=ON `
|
| 211 |
+
-DGGML_RPC=ON `
|
| 212 |
+
-DGGML_BACKEND_DL=ON `
|
| 213 |
+
-DGGML_CPU_ALL_VARIANTS=ON `
|
| 214 |
+
-DGGML_VULKAN=ON `
|
| 215 |
+
-DLLAMA_BUILD_BORINGSSL=ON
|
| 216 |
+
cmake --build build --config Release -j ${env:NUMBER_OF_PROCESSORS}
|
| 217 |
+
|
| 218 |
+
- name: Test
|
| 219 |
+
id: cmake_test
|
| 220 |
+
run: |
|
| 221 |
+
cd build
|
| 222 |
+
ctest -L main -C Release --verbose --timeout 900
|
| 223 |
+
|
| 224 |
+
- name: ccache-clear
|
| 225 |
+
uses: ./.github/actions/ccache-clear
|
| 226 |
+
env:
|
| 227 |
+
GH_TOKEN: ${{ github.token }}
|
| 228 |
+
with:
|
| 229 |
+
key: cpu-windows-2025-x64-vulkan
|
| 230 |
+
older: 5m
|
| 231 |
+
min: 1
|
| 232 |
+
dry-run: ${{ github.event_name != 'push' || github.ref != 'refs/heads/master' }}
|
.github/workflows/build-wasm.yml
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: CI (wasm)
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
workflow_dispatch: # allows manual triggering
|
| 5 |
+
push:
|
| 6 |
+
branches:
|
| 7 |
+
- master
|
| 8 |
+
paths: [
|
| 9 |
+
'.github/workflows/build-wasm.yml',
|
| 10 |
+
'**/CMakeLists.txt',
|
| 11 |
+
'**/.cmake',
|
| 12 |
+
'**/*.h',
|
| 13 |
+
'**/*.hpp',
|
| 14 |
+
'**/*.c',
|
| 15 |
+
'**/*.cpp',
|
| 16 |
+
'**/*.wgsl',
|
| 17 |
+
'**/*.tmpl',
|
| 18 |
+
'ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py'
|
| 19 |
+
]
|
| 20 |
+
|
| 21 |
+
pull_request:
|
| 22 |
+
types: [opened, synchronize, reopened]
|
| 23 |
+
paths: [
|
| 24 |
+
'.github/workflows/build-wasm.yml',
|
| 25 |
+
'**/CMakeLists.txt',
|
| 26 |
+
'**/.cmake',
|
| 27 |
+
'**/*.h',
|
| 28 |
+
'**/*.hpp',
|
| 29 |
+
'**/*.c',
|
| 30 |
+
'**/*.cpp',
|
| 31 |
+
'**/*.wgsl',
|
| 32 |
+
'**/*.tmpl',
|
| 33 |
+
'ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py'
|
| 34 |
+
]
|
| 35 |
+
|
| 36 |
+
concurrency:
|
| 37 |
+
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
| 38 |
+
cancel-in-progress: true
|
| 39 |
+
|
| 40 |
+
env:
|
| 41 |
+
GGML_NLOOP: 3
|
| 42 |
+
GGML_N_THREADS: 1
|
| 43 |
+
LLAMA_ARG_LOG_COLORS: 1
|
| 44 |
+
LLAMA_ARG_LOG_PREFIX: 1
|
| 45 |
+
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
| 46 |
+
|
| 47 |
+
jobs:
|
| 48 |
+
ubuntu-webgpu:
|
| 49 |
+
runs-on: ubuntu-24.04-arm
|
| 50 |
+
|
| 51 |
+
steps:
|
| 52 |
+
- name: Clone
|
| 53 |
+
id: checkout
|
| 54 |
+
uses: actions/checkout@v6
|
| 55 |
+
|
| 56 |
+
- name: ccache
|
| 57 |
+
uses: ggml-org/ccache-action@v1.2.24
|
| 58 |
+
with:
|
| 59 |
+
key: webgpu-ubuntu-24.04-arm-wasm
|
| 60 |
+
save: false
|
| 61 |
+
|
| 62 |
+
- name: Install Emscripten
|
| 63 |
+
run: |
|
| 64 |
+
git clone https://github.com/emscripten-core/emsdk.git
|
| 65 |
+
cd emsdk
|
| 66 |
+
./emsdk install latest
|
| 67 |
+
./emsdk activate latest
|
| 68 |
+
|
| 69 |
+
- name: Fetch emdawnwebgpu
|
| 70 |
+
run: |
|
| 71 |
+
DAWN_TAG="v20260317.182325"
|
| 72 |
+
EMDAWN_PKG="emdawnwebgpu_pkg-${DAWN_TAG}.zip"
|
| 73 |
+
echo "Downloading ${EMDAWN_PKG}"
|
| 74 |
+
curl -L -o emdawn.zip \
|
| 75 |
+
"https://github.com/google/dawn/releases/download/${DAWN_TAG}/${EMDAWN_PKG}"
|
| 76 |
+
unzip emdawn.zip
|
| 77 |
+
|
| 78 |
+
- name: ccache-buckets-restore
|
| 79 |
+
uses: ./.github/actions/ccache-buckets
|
| 80 |
+
env:
|
| 81 |
+
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
| 82 |
+
with:
|
| 83 |
+
key: webgpu-ubuntu-24.04-arm-wasm
|
| 84 |
+
folder: llama.cpp
|
| 85 |
+
hf_bucket: ggml-org/cache
|
| 86 |
+
|
| 87 |
+
- name: Build WASM WebGPU
|
| 88 |
+
run: |
|
| 89 |
+
source emsdk/emsdk_env.sh
|
| 90 |
+
emcmake cmake -B build-wasm \
|
| 91 |
+
-G "Ninja" \
|
| 92 |
+
-DCMAKE_BUILD_TYPE=Release \
|
| 93 |
+
-DGGML_WEBGPU=ON \
|
| 94 |
+
-DGGML_OPENMP=OFF \
|
| 95 |
+
-DLLAMA_OPENSSL=OFF \
|
| 96 |
+
-DEMDAWNWEBGPU_DIR=emdawnwebgpu_pkg
|
| 97 |
+
|
| 98 |
+
time cmake --build build-wasm --config Release --target test-backend-ops -j $(nproc)
|
| 99 |
+
|
| 100 |
+
- name: ccache-buckets-save
|
| 101 |
+
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
| 102 |
+
uses: ./.github/actions/ccache-buckets
|
| 103 |
+
env:
|
| 104 |
+
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
| 105 |
+
with:
|
| 106 |
+
key: webgpu-ubuntu-24.04-arm-wasm
|
| 107 |
+
folder: llama.cpp
|
| 108 |
+
evict-old-files: 1d
|
| 109 |
+
hf_bucket: ggml-org/cache
|
| 110 |
+
save: true
|
.github/workflows/build-webgpu.yml
ADDED
|
@@ -0,0 +1,195 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
name: CI (webgpu)
|
| 2 |
+
|
| 3 |
+
on:
|
| 4 |
+
workflow_dispatch: # allows manual triggering
|
| 5 |
+
push:
|
| 6 |
+
branches:
|
| 7 |
+
- master
|
| 8 |
+
paths: [
|
| 9 |
+
'.github/workflows/build-webgpu.yml',
|
| 10 |
+
'**/CMakeLists.txt',
|
| 11 |
+
'**/.cmake',
|
| 12 |
+
'**/*.h',
|
| 13 |
+
'**/*.hpp',
|
| 14 |
+
'**/*.c',
|
| 15 |
+
'**/*.cpp',
|
| 16 |
+
'**/*.wgsl',
|
| 17 |
+
'**/*.tmpl',
|
| 18 |
+
'ggml/src/ggml-webgpu/wgsl-shaders/embed_wgsl.py'
|
| 19 |
+
]
|
| 20 |
+
|
| 21 |
+
pull_request:
|
| 22 |
+
types: [opened, synchronize, reopened]
|
| 23 |
+
paths: [
|
| 24 |
+
'.github/workflows/build-webgpu.yml',
|
| 25 |
+
'ggml/src/ggml-webgpu/**'
|
| 26 |
+
]
|
| 27 |
+
|
| 28 |
+
concurrency:
|
| 29 |
+
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}
|
| 30 |
+
cancel-in-progress: true
|
| 31 |
+
|
| 32 |
+
env:
|
| 33 |
+
GGML_NLOOP: 3
|
| 34 |
+
GGML_N_THREADS: 1
|
| 35 |
+
LLAMA_ARG_LOG_COLORS: 1
|
| 36 |
+
LLAMA_ARG_LOG_PREFIX: 1
|
| 37 |
+
LLAMA_ARG_LOG_TIMESTAMPS: 1
|
| 38 |
+
|
| 39 |
+
jobs:
|
| 40 |
+
format:
|
| 41 |
+
runs-on: ubuntu-24.04
|
| 42 |
+
|
| 43 |
+
steps:
|
| 44 |
+
- name: Clone
|
| 45 |
+
uses: actions/checkout@v6
|
| 46 |
+
|
| 47 |
+
- name: Install clang-format 22
|
| 48 |
+
run: |
|
| 49 |
+
wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key |
|
| 50 |
+
sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc > /dev/null
|
| 51 |
+
sudo add-apt-repository -y \
|
| 52 |
+
"deb http://apt.llvm.org/noble/ llvm-toolchain-noble-22 main"
|
| 53 |
+
sudo apt-get update
|
| 54 |
+
sudo apt-get install -y clang-format-22
|
| 55 |
+
|
| 56 |
+
- name: Check formatting
|
| 57 |
+
run: |
|
| 58 |
+
find ggml/src/ggml-webgpu \
|
| 59 |
+
-type f \( -name '*.cpp' -o -name '*.hpp' -o -name '*.h' \) \
|
| 60 |
+
-print0 |
|
| 61 |
+
xargs -0 clang-format-22 --dry-run --Werror
|
| 62 |
+
|
| 63 |
+
macos:
|
| 64 |
+
runs-on: macos-latest
|
| 65 |
+
|
| 66 |
+
steps:
|
| 67 |
+
- name: Clone
|
| 68 |
+
id: checkout
|
| 69 |
+
uses: actions/checkout@v6
|
| 70 |
+
|
| 71 |
+
- name: ccache
|
| 72 |
+
uses: ggml-org/ccache-action@v1.2.24
|
| 73 |
+
with:
|
| 74 |
+
key: webgpu-macos-latest
|
| 75 |
+
save: false
|
| 76 |
+
|
| 77 |
+
- name: Dawn Dependency
|
| 78 |
+
id: dawn-depends
|
| 79 |
+
run: |
|
| 80 |
+
DAWN_VERSION="v20260317.182325"
|
| 81 |
+
DAWN_OWNER="google"
|
| 82 |
+
DAWN_REPO="dawn"
|
| 83 |
+
DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-macos-latest-Release"
|
| 84 |
+
echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
|
| 85 |
+
curl -L -o artifact.tar.gz \
|
| 86 |
+
"https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
|
| 87 |
+
mkdir dawn
|
| 88 |
+
tar -xvf artifact.tar.gz -C dawn --strip-components=1
|
| 89 |
+
|
| 90 |
+
- name: ccache-buckets-restore
|
| 91 |
+
uses: ./.github/actions/ccache-buckets
|
| 92 |
+
env:
|
| 93 |
+
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
| 94 |
+
with:
|
| 95 |
+
key: webgpu-macos-latest
|
| 96 |
+
folder: llama.cpp
|
| 97 |
+
hf_bucket: ggml-org/cache
|
| 98 |
+
|
| 99 |
+
- name: Build
|
| 100 |
+
id: cmake_build
|
| 101 |
+
run: |
|
| 102 |
+
export CMAKE_PREFIX_PATH=dawn
|
| 103 |
+
cmake -B build -G "Ninja" -DCMAKE_BUILD_TYPE=Release -DGGML_WEBGPU=ON -DGGML_METAL=OFF -DGGML_BLAS=OFF
|
| 104 |
+
time cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
|
| 105 |
+
|
| 106 |
+
- name: ccache-buckets-save
|
| 107 |
+
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
| 108 |
+
uses: ./.github/actions/ccache-buckets
|
| 109 |
+
env:
|
| 110 |
+
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
| 111 |
+
with:
|
| 112 |
+
key: webgpu-macos-latest
|
| 113 |
+
folder: llama.cpp
|
| 114 |
+
evict-old-files: 1d
|
| 115 |
+
hf_bucket: ggml-org/cache
|
| 116 |
+
save: true
|
| 117 |
+
|
| 118 |
+
- name: Test
|
| 119 |
+
id: cmake_test
|
| 120 |
+
run: |
|
| 121 |
+
cd build
|
| 122 |
+
ctest -L main --verbose --timeout 900
|
| 123 |
+
|
| 124 |
+
ubuntu:
|
| 125 |
+
runs-on: ubuntu-24.04
|
| 126 |
+
|
| 127 |
+
steps:
|
| 128 |
+
- name: Clone
|
| 129 |
+
id: checkout
|
| 130 |
+
uses: actions/checkout@v6
|
| 131 |
+
|
| 132 |
+
- name: ccache
|
| 133 |
+
uses: ggml-org/ccache-action@v1.2.24
|
| 134 |
+
with:
|
| 135 |
+
key: webgpu-ubuntu-24.04
|
| 136 |
+
save: false
|
| 137 |
+
|
| 138 |
+
- name: Dependencies
|
| 139 |
+
id: depends
|
| 140 |
+
run: |
|
| 141 |
+
sudo add-apt-repository -y ppa:kisak/kisak-mesa
|
| 142 |
+
sudo apt-get update -y
|
| 143 |
+
sudo apt-get install -y build-essential mesa-vulkan-drivers \
|
| 144 |
+
libxcb-xinput0 libxcb-xinerama0 libxcb-cursor-dev libssl-dev
|
| 145 |
+
|
| 146 |
+
- name: Dawn Dependency
|
| 147 |
+
id: dawn-depends
|
| 148 |
+
run: |
|
| 149 |
+
sudo apt-get install -y libxrandr-dev libxinerama-dev libxcursor-dev mesa-common-dev libx11-xcb-dev libxi-dev
|
| 150 |
+
DAWN_VERSION="v20260317.182325"
|
| 151 |
+
DAWN_OWNER="google"
|
| 152 |
+
DAWN_REPO="dawn"
|
| 153 |
+
DAWN_ASSET_NAME="Dawn-18eb229ef5f707c1464cc581252e7603c73a3ef0-ubuntu-latest-Release"
|
| 154 |
+
echo "Fetching release asset from https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
|
| 155 |
+
curl -L -o artifact.tar.gz \
|
| 156 |
+
"https://github.com/google/dawn/releases/download/${DAWN_VERSION}/${DAWN_ASSET_NAME}.tar.gz"
|
| 157 |
+
mkdir dawn
|
| 158 |
+
tar -xvf artifact.tar.gz -C dawn --strip-components=1
|
| 159 |
+
|
| 160 |
+
- name: ccache-buckets-restore
|
| 161 |
+
uses: ./.github/actions/ccache-buckets
|
| 162 |
+
env:
|
| 163 |
+
HF_TOKEN: ${{ secrets.HF_TOKEN_CI }}
|
| 164 |
+
with:
|
| 165 |
+
key: webgpu-ubuntu-24.04
|
| 166 |
+
folder: llama.cpp
|
| 167 |
+
hf_bucket: ggml-org/cache
|
| 168 |
+
|
| 169 |
+
- name: Build
|
| 170 |
+
id: cmake_build
|
| 171 |
+
run: |
|
| 172 |
+
export Dawn_DIR=dawn/lib64/cmake/Dawn
|
| 173 |
+
cmake -B build \
|
| 174 |
+
-DGGML_WEBGPU=ON
|
| 175 |
+
time cmake --build build --config Release -j $(nproc)
|
| 176 |
+
|
| 177 |
+
- name: ccache-buckets-save
|
| 178 |
+
if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' }}
|
| 179 |
+
uses: ./.github/actions/ccache-buckets
|
| 180 |
+
env:
|
| 181 |
+
HF_TOKEN: ${{ secrets.HF_TOKEN_CACHE_OUTPUT }}
|
| 182 |
+
with:
|
| 183 |
+
key: webgpu-ubuntu-24.04
|
| 184 |
+
folder: llama.cpp
|
| 185 |
+
evict-old-files: 1d
|
| 186 |
+
hf_bucket: ggml-org/cache
|
| 187 |
+
save: true
|
| 188 |
+
|
| 189 |
+
- name: Test
|
| 190 |
+
id: cmake_test
|
| 191 |
+
run: |
|
| 192 |
+
cd build
|
| 193 |
+
# This is using llvmpipe and runs slower than other backends
|
| 194 |
+
# test-backend-ops is too slow on llvmpipe, skip it
|
| 195 |
+
ctest -L main -E test-backend-ops --verbose --timeout 900
|
cmake/build-info.cmake
ADDED
|
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
set(BUILD_NUMBER 0)
|
| 2 |
+
set(BUILD_COMMIT "unknown")
|
| 3 |
+
set(BUILD_COMPILER "unknown")
|
| 4 |
+
set(BUILD_TARGET "unknown")
|
| 5 |
+
|
| 6 |
+
# Look for git
|
| 7 |
+
find_package(Git)
|
| 8 |
+
if(NOT Git_FOUND)
|
| 9 |
+
find_program(GIT_EXECUTABLE NAMES git git.exe)
|
| 10 |
+
if(GIT_EXECUTABLE)
|
| 11 |
+
set(Git_FOUND TRUE)
|
| 12 |
+
message(STATUS "Found Git: ${GIT_EXECUTABLE}")
|
| 13 |
+
else()
|
| 14 |
+
message(WARNING "Git not found. Build info will not be accurate.")
|
| 15 |
+
endif()
|
| 16 |
+
endif()
|
| 17 |
+
|
| 18 |
+
# Get the commit count and hash
|
| 19 |
+
if(Git_FOUND)
|
| 20 |
+
execute_process(
|
| 21 |
+
COMMAND ${GIT_EXECUTABLE} rev-parse --short HEAD
|
| 22 |
+
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
|
| 23 |
+
OUTPUT_VARIABLE HEAD
|
| 24 |
+
OUTPUT_STRIP_TRAILING_WHITESPACE
|
| 25 |
+
RESULT_VARIABLE RES
|
| 26 |
+
)
|
| 27 |
+
if (RES EQUAL 0)
|
| 28 |
+
set(BUILD_COMMIT ${HEAD})
|
| 29 |
+
endif()
|
| 30 |
+
execute_process(
|
| 31 |
+
COMMAND ${GIT_EXECUTABLE} rev-list --count HEAD
|
| 32 |
+
WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
|
| 33 |
+
OUTPUT_VARIABLE COUNT
|
| 34 |
+
OUTPUT_STRIP_TRAILING_WHITESPACE
|
| 35 |
+
RESULT_VARIABLE RES
|
| 36 |
+
)
|
| 37 |
+
if (RES EQUAL 0)
|
| 38 |
+
set(BUILD_NUMBER ${COUNT})
|
| 39 |
+
endif()
|
| 40 |
+
endif()
|
| 41 |
+
|
| 42 |
+
set(BUILD_COMPILER "${CMAKE_C_COMPILER_ID} ${CMAKE_C_COMPILER_VERSION}")
|
| 43 |
+
|
| 44 |
+
if(CMAKE_VS_PLATFORM_NAME)
|
| 45 |
+
set(BUILD_TARGET ${CMAKE_VS_PLATFORM_NAME})
|
| 46 |
+
else()
|
| 47 |
+
set(BUILD_TARGET "${CMAKE_SYSTEM_NAME} ${CMAKE_SYSTEM_PROCESSOR}")
|
| 48 |
+
endif()
|
common/build-info.cpp.in
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#include "build-info.h"
|
| 2 |
+
|
| 3 |
+
#include <cstdio>
|
| 4 |
+
#include <string>
|
| 5 |
+
|
| 6 |
+
int LLAMA_BUILD_NUMBER = @LLAMA_BUILD_NUMBER@;
|
| 7 |
+
char const * LLAMA_COMMIT = "@LLAMA_BUILD_COMMIT@";
|
| 8 |
+
char const * LLAMA_COMPILER = "@BUILD_COMPILER@";
|
| 9 |
+
char const * LLAMA_BUILD_TARGET = "@BUILD_TARGET@";
|
| 10 |
+
|
| 11 |
+
int llama_build_number(void) {
|
| 12 |
+
return LLAMA_BUILD_NUMBER;
|
| 13 |
+
}
|
| 14 |
+
|
| 15 |
+
const char * llama_commit(void) {
|
| 16 |
+
return LLAMA_COMMIT;
|
| 17 |
+
}
|
| 18 |
+
|
| 19 |
+
const char * llama_compiler(void) {
|
| 20 |
+
return LLAMA_COMPILER;
|
| 21 |
+
}
|
| 22 |
+
|
| 23 |
+
const char * llama_build_target(void) {
|
| 24 |
+
return LLAMA_BUILD_TARGET;
|
| 25 |
+
}
|
| 26 |
+
|
| 27 |
+
const char * llama_build_info(void) {
|
| 28 |
+
static std::string s = "b" + std::to_string(LLAMA_BUILD_NUMBER) + "-" + LLAMA_COMMIT;
|
| 29 |
+
return s.c_str();
|
| 30 |
+
}
|
| 31 |
+
|
| 32 |
+
void llama_print_build_info(const char * llama_version, FILE * stream) {
|
| 33 |
+
fprintf(stream, "version: %s (build %d, commit %s)\n", llama_version, llama_build_number(), llama_commit());
|
| 34 |
+
fprintf(stream, "built with %s for %s\n", llama_compiler(), llama_build_target());
|
| 35 |
+
}
|
common/build-info.h
ADDED
|
@@ -0,0 +1,13 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#pragma once
|
| 2 |
+
|
| 3 |
+
#include <cstdio>
|
| 4 |
+
|
| 5 |
+
int llama_build_number(void);
|
| 6 |
+
|
| 7 |
+
const char * llama_commit(void);
|
| 8 |
+
const char * llama_compiler(void);
|
| 9 |
+
|
| 10 |
+
const char * llama_build_target(void);
|
| 11 |
+
const char * llama_build_info(void);
|
| 12 |
+
|
| 13 |
+
void llama_print_build_info(const char *, FILE * = stderr);
|
docs/build-riscv64-spacemit.md
ADDED
|
@@ -0,0 +1,110 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
> [!IMPORTANT]
|
| 2 |
+
> This build documentation is specific only to RISC-V SpacemiT SOCs.
|
| 3 |
+
|
| 4 |
+
## Build llama.cpp locally (for riscv64)
|
| 5 |
+
|
| 6 |
+
1. Prepare Toolchain For RISCV
|
| 7 |
+
~~~
|
| 8 |
+
wget https://github.com/spacemit-com/toolchain/releases/download/v1.2.4/spacemit-toolchain-linux-glibc-x86_64-v1.2.4.tar.xz
|
| 9 |
+
~~~
|
| 10 |
+
|
| 11 |
+
2. Build
|
| 12 |
+
Below is the build script: it requires utilizing RISC-V vector instructions for acceleration. Ensure the `GGML_CPU_RISCV64_SPACEMIT` compilation option is enabled. The currently supported optimization version is `RISCV64_SPACEMIT_IME1` and `RISCV64_SPACEMIT_IME2`, corresponding to the `RISCV64_SPACEMIT_IME_SPEC` compilation option. Compiler configurations are defined in the `riscv64-spacemit-linux-gnu-gcc.cmake` file. Please ensure you have installed the RISC-V compiler and set the environment variable via `export RISCV_ROOT_PATH={your_compiler_path}`.
|
| 13 |
+
```bash
|
| 14 |
+
|
| 15 |
+
cmake -B build \
|
| 16 |
+
-DCMAKE_BUILD_TYPE=Release \
|
| 17 |
+
-DGGML_CPU_RISCV64_SPACEMIT=ON \
|
| 18 |
+
-DGGML_CPU_REPACK=OFF \
|
| 19 |
+
-DLLAMA_OPENSSL=OFF \
|
| 20 |
+
-DGGML_RVV=ON \
|
| 21 |
+
-DGGML_RV_ZVFH=ON \
|
| 22 |
+
-DGGML_RV_ZFH=ON \
|
| 23 |
+
-DGGML_RV_ZICBOP=ON \
|
| 24 |
+
-DGGML_RV_ZIHINTPAUSE=ON \
|
| 25 |
+
-DGGML_RV_ZBA=ON \
|
| 26 |
+
-DCMAKE_TOOLCHAIN_FILE=${PWD}/cmake/riscv64-spacemit-linux-gnu-gcc.cmake \
|
| 27 |
+
-DCMAKE_INSTALL_PREFIX=build/installed
|
| 28 |
+
|
| 29 |
+
cmake --build build --parallel $(nproc) --config Release
|
| 30 |
+
|
| 31 |
+
pushd build
|
| 32 |
+
make install
|
| 33 |
+
popd
|
| 34 |
+
```
|
| 35 |
+
|
| 36 |
+
## Simulation
|
| 37 |
+
You can use QEMU to perform emulation on non-RISC-V architectures.
|
| 38 |
+
|
| 39 |
+
1. Download QEMU
|
| 40 |
+
~~~
|
| 41 |
+
wget https://archive.spacemit.com/spacemit-ai/qemu/jdsk-qemu-v0.0.14.tar.gz
|
| 42 |
+
~~~
|
| 43 |
+
|
| 44 |
+
2. Run Simulation
|
| 45 |
+
After build your llama.cpp, you can run the executable file via QEMU for simulation, for example:
|
| 46 |
+
~~~
|
| 47 |
+
export QEMU_ROOT_PATH={your QEMU file path}
|
| 48 |
+
export RISCV_ROOT_PATH_IME1={your RISC-V compiler path}
|
| 49 |
+
|
| 50 |
+
${QEMU_ROOT_PATH}/bin/qemu-riscv64 -L ${RISCV_ROOT_PATH_IME1}/sysroot -cpu max,vlen=256,elen=64,vext_spec=v1.0 ${PWD}/build/bin/llama-cli -m ${PWD}/models/Qwen2.5-0.5B-Instruct-Q4_0.gguf -t 1
|
| 51 |
+
~~~
|
| 52 |
+
|
| 53 |
+
## Quantization Support For Matrix
|
| 54 |
+
|
| 55 |
+
| Quantization Type | X60 | A100 |
|
| 56 |
+
| ---: | ---: | ---: |
|
| 57 |
+
| Q2_K | | :heavy_check_mark: |
|
| 58 |
+
| Q3_K | | :heavy_check_mark: |
|
| 59 |
+
| Q4_0 | :heavy_check_mark: | :heavy_check_mark: |
|
| 60 |
+
| Q4_1 | :heavy_check_mark: | :heavy_check_mark: |
|
| 61 |
+
| Q4_K | :heavy_check_mark: | :heavy_check_mark: |
|
| 62 |
+
| Q5_0 | | :heavy_check_mark: |
|
| 63 |
+
| Q5_1 | | :heavy_check_mark: |
|
| 64 |
+
| Q5_K | | :heavy_check_mark: |
|
| 65 |
+
| Q6_K | | :heavy_check_mark: |
|
| 66 |
+
| Q8_0 | | :heavy_check_mark: |
|
| 67 |
+
|
| 68 |
+
|
| 69 |
+
## Performance
|
| 70 |
+
* Spacemit(R) X60
|
| 71 |
+
~~~
|
| 72 |
+
model name : Spacemit(R) X60
|
| 73 |
+
isa : rv64imafdcv_zicbom_zicboz_zicntr_zicond_zicsr_zifencei_zihintpause_zihpm_zfh_zfhmin_zca_zcd_zba_zbb_zbc_zbs_zkt_zve32f_zve32x_zve64d_zve64f_zve64x_zvfh_zvfhmin_zvkt_sscofpmf_sstc_svinval_svnapot_svpbmt
|
| 74 |
+
mmu : sv39
|
| 75 |
+
uarch : spacemit,x60
|
| 76 |
+
mvendorid : 0x710
|
| 77 |
+
marchid : 0x8000000058000001
|
| 78 |
+
~~~
|
| 79 |
+
|
| 80 |
+
| model | size | params | backend | threads | n_ubatch | fa | mmap | test | t/s |
|
| 81 |
+
| ------------------------------ | ---------: | ---------: | ---------- | ------: | -------: | -: | ---: | --------------: | -------------------: |
|
| 82 |
+
| qwen35 2B Q4_1 | 1.19 GiB | 1.88 B | CPU | 4 | 128 | 1 | 0 | pp128 | 10.32 ± 0.02 |
|
| 83 |
+
| qwen35 2B Q4_1 | 1.19 GiB | 1.88 B | CPU | 4 | 128 | 1 | 0 | tg128 | 3.07 ± 0.01 |
|
| 84 |
+
| qwen3 0.6B Q4_0 | 358.78 MiB | 596.05 M | CPU | 4 | 128 | 1 | 0 | pp128 | 49.15 ± 0.25 |
|
| 85 |
+
| qwen3 0.6B Q4_0 | 358.78 MiB | 596.05 M | CPU | 4 | 128 | 1 | 0 | tg128 | 11.73 ± 0.02 |
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
* Spacemit(R) A100
|
| 89 |
+
~~~
|
| 90 |
+
model name : Spacemit(R) A100
|
| 91 |
+
isa : rv64imafdcvh_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintntl_zihintpause_zihpm_zimop_zaamo_zalrsc_zawrs_zfa_zfh_zfhmin_zca_zcb_zcd_zcmop_zba_zbb_zbc_zbs_zkt_zvbb_zvbc_zve32f_zve32x_zve64d_zve64f_zve64x_zvfh_zvfhmin_zvkb_zvkg_zvkned_zvknha_zvknhb_zvksed_zvksh_zvkt_smaia_smstateen_ssaia_sscofpmf_sstc_svinval_svnapot_svpbmt_sdtrig
|
| 92 |
+
mmu : sv39
|
| 93 |
+
mvendorid : 0x710
|
| 94 |
+
marchid : 0x8000000041000002
|
| 95 |
+
mimpid : 0x10000000d5686200
|
| 96 |
+
hart isa : rv64imafdcv_zicbom_zicbop_zicboz_zicntr_zicond_zicsr_zifencei_zihintntl_zihintpause_zihpm_zimop_zaamo_zalrsc_zawrs_zfa_zfh_zfhmin_zca_zcb_zcd_zcmop_zba_zbb_zbc_zbs_zkt_zvbb_zvbc_zve32f_zve32x_zve64d_zve64f_zve64x_zvfh_zvfhmin_zvkb_zvkg_zvkned_zvknha_zvknhb_zvksed_zvksh_zvkt_smaia_smstateen_ssaia_sscofpmf_sstc_svinval_svnapot_svpbmt_sdtrig
|
| 97 |
+
~~~
|
| 98 |
+
|
| 99 |
+
| model | size | params | backend | threads | n_ubatch | fa | mmap | test | t/s |
|
| 100 |
+
| ------------------------------ | ---------: | ---------: | ---------- | ------: | -------: | -: | ---: | --------------: | -------------------: |
|
| 101 |
+
| qwen3 0.6B Q4_0 | 358.78 MiB | 596.05 M | CPU | 8 | 128 | 1 | 0 | pp128 | 565.83 ± 0.31 |
|
| 102 |
+
| qwen3 0.6B Q4_0 | 358.78 MiB | 596.05 M | CPU | 8 | 128 | 1 | 0 | tg128 | 55.77 ± 0.02 |
|
| 103 |
+
| qwen3 4B Q4_0 | 2.21 GiB | 4.02 B | CPU | 8 | 128 | 1 | 0 | pp128 | 79.74 ± 0.04 |
|
| 104 |
+
| qwen3 4B Q4_0 | 2.21 GiB | 4.02 B | CPU | 8 | 128 | 1 | 0 | tg128 | 11.29 ± 0.00 |
|
| 105 |
+
| qwen3moe 30B.A3B Q4_0 | 16.18 GiB | 30.53 B | CPU | 8 | 128 | 1 | 0 | pp128 | 57.88 ± 0.31 |
|
| 106 |
+
| qwen3moe 30B.A3B Q4_0 | 16.18 GiB | 30.53 B | CPU | 8 | 128 | 1 | 0 | tg128 | 12.79 ± 0.00 |
|
| 107 |
+
| qwen35 2B Q4_1 | 1.19 GiB | 1.88 B | CPU | 8 | 128 | 1 | 0 | pp128 | 115.23 ± 0.04 |
|
| 108 |
+
| qwen35 2B Q4_1 | 1.19 GiB | 1.88 B | CPU | 8 | 128 | 1 | 0 | tg128 | 16.49 ± 0.01 |
|
| 109 |
+
| gemma4 E4B Q4_K - Medium | 4.76 GiB | 7.52 B | CPU | 8 | 128 | 1 | 0 | pp128 | 21.13 ± 0.01 |
|
| 110 |
+
| gemma4 E4B Q4_K - Medium | 4.76 GiB | 7.52 B | CPU | 8 | 128 | 1 | 0 | tg128 | 5.66 ± 0.00 |
|
docs/build-s390x.md
ADDED
|
@@ -0,0 +1,275 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
> [!IMPORTANT]
|
| 2 |
+
> This build documentation is specific only to IBM Z & LinuxONE mainframes (s390x). You can find the build documentation for other architectures: [build.md](build.md).
|
| 3 |
+
|
| 4 |
+
# Build llama.cpp locally (for s390x)
|
| 5 |
+
|
| 6 |
+
The main product of this project is the `llama` library. Its C-style interface can be found in [include/llama.h](../include/llama.h).
|
| 7 |
+
|
| 8 |
+
The project also includes many example programs and tools using the `llama` library. The examples range from simple, minimal code snippets to sophisticated sub-projects such as an OpenAI-compatible HTTP server.
|
| 9 |
+
|
| 10 |
+
**To get the code:**
|
| 11 |
+
|
| 12 |
+
```bash
|
| 13 |
+
git clone https://github.com/ggml-org/llama.cpp
|
| 14 |
+
cd llama.cpp
|
| 15 |
+
```
|
| 16 |
+
|
| 17 |
+
## CPU Build with BLAS
|
| 18 |
+
|
| 19 |
+
Building llama.cpp with BLAS support is highly recommended as it has shown to provide performance improvements. Make sure to have OpenBLAS installed in your environment.
|
| 20 |
+
|
| 21 |
+
```bash
|
| 22 |
+
cmake -S . -B build \
|
| 23 |
+
-DCMAKE_BUILD_TYPE=Release \
|
| 24 |
+
-DGGML_BLAS=ON \
|
| 25 |
+
-DGGML_BLAS_VENDOR=OpenBLAS
|
| 26 |
+
|
| 27 |
+
cmake --build build --config Release -j $(nproc)
|
| 28 |
+
```
|
| 29 |
+
|
| 30 |
+
**Notes**:
|
| 31 |
+
|
| 32 |
+
- For faster repeated compilation, install [ccache](https://ccache.dev/)
|
| 33 |
+
- By default, VXE/VXE2 is enabled. To disable it (not recommended):
|
| 34 |
+
|
| 35 |
+
```bash
|
| 36 |
+
cmake -S . -B build \
|
| 37 |
+
-DCMAKE_BUILD_TYPE=Release \
|
| 38 |
+
-DGGML_BLAS=ON \
|
| 39 |
+
-DGGML_BLAS_VENDOR=OpenBLAS \
|
| 40 |
+
-DGGML_VXE=OFF
|
| 41 |
+
|
| 42 |
+
cmake --build build --config Release -j $(nproc)
|
| 43 |
+
```
|
| 44 |
+
|
| 45 |
+
- For debug builds:
|
| 46 |
+
|
| 47 |
+
```bash
|
| 48 |
+
cmake -S . -B build \
|
| 49 |
+
-DCMAKE_BUILD_TYPE=Debug \
|
| 50 |
+
-DGGML_BLAS=ON \
|
| 51 |
+
-DGGML_BLAS_VENDOR=OpenBLAS
|
| 52 |
+
cmake --build build --config Debug -j $(nproc)
|
| 53 |
+
```
|
| 54 |
+
|
| 55 |
+
- For static builds, add `-DBUILD_SHARED_LIBS=OFF`:
|
| 56 |
+
|
| 57 |
+
```bash
|
| 58 |
+
cmake -S . -B build \
|
| 59 |
+
-DCMAKE_BUILD_TYPE=Release \
|
| 60 |
+
-DGGML_BLAS=ON \
|
| 61 |
+
-DGGML_BLAS_VENDOR=OpenBLAS \
|
| 62 |
+
-DBUILD_SHARED_LIBS=OFF
|
| 63 |
+
|
| 64 |
+
cmake --build build --config Release -j $(nproc)
|
| 65 |
+
```
|
| 66 |
+
|
| 67 |
+
## IBM zDNN Accelerator
|
| 68 |
+
|
| 69 |
+
This provides acceleration using the IBM zAIU co-processor located in the Telum I and Telum II processors. Make sure to have the [IBM zDNN library](https://github.com/IBM/zDNN) installed.
|
| 70 |
+
|
| 71 |
+
#### Compile from source from IBM
|
| 72 |
+
|
| 73 |
+
You may find the official build instructions here: [Building and Installing zDNN](https://github.com/IBM/zDNN?tab=readme-ov-file#building-and-installing-zdnn)
|
| 74 |
+
|
| 75 |
+
### Compilation
|
| 76 |
+
|
| 77 |
+
```bash
|
| 78 |
+
cmake -S . -B build \
|
| 79 |
+
-DCMAKE_BUILD_TYPE=Release \
|
| 80 |
+
-DGGML_ZDNN=ON
|
| 81 |
+
cmake --build build --config Release -j$(nproc)
|
| 82 |
+
```
|
| 83 |
+
|
| 84 |
+
## Getting GGUF Models
|
| 85 |
+
|
| 86 |
+
All models need to be converted to Big-Endian. You can achieve this in three cases:
|
| 87 |
+
|
| 88 |
+
1. **Use pre-converted models verified for use on IBM Z & LinuxONE (easiest)**
|
| 89 |
+
|
| 90 |
+

|
| 91 |
+
|
| 92 |
+
You can find popular models pre-converted and verified at [s390x Verified Models](https://huggingface.co/collections/taronaeo/s390x-verified-models-672765393af438d0ccb72a08) or [s390x Runnable Models](https://huggingface.co/collections/taronaeo/s390x-runnable-models-686e951824198df12416017e).
|
| 93 |
+
|
| 94 |
+
These models have already been converted from `safetensors` to `GGUF` Big-Endian and their respective tokenizers verified to run correctly on IBM z15 and later system.
|
| 95 |
+
|
| 96 |
+
2. **Convert safetensors model to GGUF Big-Endian directly (recommended)**
|
| 97 |
+
|
| 98 |
+

|
| 99 |
+
|
| 100 |
+
The model you are trying to convert must be in `safetensors` file format (for example [IBM Granite 3.3 2B](https://huggingface.co/ibm-granite/granite-3.3-2b-instruct)). Make sure you have downloaded the model repository for this case.
|
| 101 |
+
|
| 102 |
+
Ensure that you have installed the required packages in advance
|
| 103 |
+
|
| 104 |
+
```bash
|
| 105 |
+
pip3 install -r requirements.txt
|
| 106 |
+
```
|
| 107 |
+
|
| 108 |
+
Convert the `safetensors` model to `GGUF`
|
| 109 |
+
|
| 110 |
+
```bash
|
| 111 |
+
python3 convert_hf_to_gguf.py \
|
| 112 |
+
--outfile model-name-be.f16.gguf \
|
| 113 |
+
--outtype f16 \
|
| 114 |
+
--bigendian \
|
| 115 |
+
model-directory/
|
| 116 |
+
```
|
| 117 |
+
|
| 118 |
+
For example,
|
| 119 |
+
|
| 120 |
+
```bash
|
| 121 |
+
python3 convert_hf_to_gguf.py \
|
| 122 |
+
--outfile granite-3.3-2b-instruct-be.f16.gguf \
|
| 123 |
+
--outtype f16 \
|
| 124 |
+
--bigendian \
|
| 125 |
+
granite-3.3-2b-instruct/
|
| 126 |
+
```
|
| 127 |
+
|
| 128 |
+
3. **Convert existing GGUF Little-Endian model to Big-Endian**
|
| 129 |
+
|
| 130 |
+

|
| 131 |
+
|
| 132 |
+
The model you are trying to convert must be in `gguf` file format (for example [IBM Granite 3.3 2B GGUF](https://huggingface.co/ibm-granite/granite-3.3-2b-instruct-GGUF)). Make sure you have downloaded the model file for this case.
|
| 133 |
+
|
| 134 |
+
```bash
|
| 135 |
+
python3 gguf-py/gguf/scripts/gguf_convert_endian.py model-name.f16.gguf BIG
|
| 136 |
+
```
|
| 137 |
+
|
| 138 |
+
For example,
|
| 139 |
+
|
| 140 |
+
```bash
|
| 141 |
+
python3 gguf-py/gguf/scripts/gguf_convert_endian.py granite-3.3-2b-instruct-le.f16.gguf BIG
|
| 142 |
+
mv granite-3.3-2b-instruct-le.f16.gguf granite-3.3-2b-instruct-be.f16.gguf
|
| 143 |
+
```
|
| 144 |
+
|
| 145 |
+
**Notes:**
|
| 146 |
+
|
| 147 |
+
- The GGUF endian conversion script may not support all data types at the moment and may fail for some models/quantizations. When that happens, please try manually converting the safetensors model to GGUF Big-Endian via Step 2.
|
| 148 |
+
|
| 149 |
+
## IBM Accelerators
|
| 150 |
+
|
| 151 |
+
### 1. SIMD Acceleration
|
| 152 |
+
|
| 153 |
+
Only available in IBM z15/LinuxONE 3 or later system with the `-DGGML_VXE=ON` (turned on by default) compile flag. No hardware acceleration is possible with llama.cpp with older systems, such as IBM z14/arch12. In such systems, the APIs can still run but will use a scalar implementation.
|
| 154 |
+
|
| 155 |
+
### 2. zDNN Accelerator (WIP)
|
| 156 |
+
|
| 157 |
+
Only available in IBM z17/LinuxONE 5 or later system with the `-DGGML_ZDNN=ON` compile flag. No hardware acceleration is possible with llama.cpp with older systems, such as IBM z15/arch13. In such systems, the APIs will default back to CPU routines.
|
| 158 |
+
|
| 159 |
+
### 3. Spyre Accelerator
|
| 160 |
+
|
| 161 |
+
_Only available with IBM z17 / LinuxONE 5 or later system. No support currently available._
|
| 162 |
+
|
| 163 |
+
## Performance Tuning
|
| 164 |
+
|
| 165 |
+
### 1. Virtualization Setup
|
| 166 |
+
|
| 167 |
+
It is strongly recommended to use only LPAR (Type-1) virtualization to get the most performance.
|
| 168 |
+
|
| 169 |
+
Note: Type-2 virtualization is not supported at the moment, while you can get it running, the performance will not be the best.
|
| 170 |
+
|
| 171 |
+
### 2. IFL (Core) Count
|
| 172 |
+
|
| 173 |
+
It is recommended to allocate a minimum of 8 shared IFLs assigned to the LPAR. Increasing the IFL count past 8 shared IFLs will only improve Prompt Processing performance but not Token Generation.
|
| 174 |
+
|
| 175 |
+
Note: IFL count does not equate to vCPU count.
|
| 176 |
+
|
| 177 |
+
### 3. SMT vs NOSMT (Simultaneous Multithreading)
|
| 178 |
+
|
| 179 |
+
It is strongly recommended to disable SMT via the kernel boot parameters as it negatively affects performance. Please refer to your Linux distribution's guide on disabling SMT via kernel boot parameters.
|
| 180 |
+
|
| 181 |
+
### 4. BLAS vs NOBLAS
|
| 182 |
+
|
| 183 |
+
IBM VXE/VXE2 SIMD acceleration depends on the BLAS implementation. It is strongly recommended to use BLAS.
|
| 184 |
+
|
| 185 |
+
## Frequently Asked Questions (FAQ)
|
| 186 |
+
|
| 187 |
+
1. I'm getting the following error message while trying to load a model: `gguf_init_from_file_impl: failed to load model: this GGUF file version 50331648 is extremely large, is there a mismatch between the host and model endianness?`
|
| 188 |
+
|
| 189 |
+
Answer: Please ensure that the model you have downloaded/converted is GGUFv3 Big-Endian. These models are usually denoted with the `-be` suffix, i.e., `granite-3.3-2b-instruct-be.F16.gguf`.
|
| 190 |
+
|
| 191 |
+
You may refer to the [Getting GGUF Models](#getting-gguf-models) section to manually convert a `safetensors` model to `GGUF` Big Endian.
|
| 192 |
+
|
| 193 |
+
2. I'm getting extremely poor performance when running inference on a model
|
| 194 |
+
|
| 195 |
+
Answer: Please refer to the [Appendix B: SIMD Support Matrix](#appendix-b-simd-support-matrix) to check if your model quantization is supported by SIMD acceleration.
|
| 196 |
+
|
| 197 |
+
3. I'm building on IBM z17 and getting the following error messages: `invalid switch -march=z17`
|
| 198 |
+
|
| 199 |
+
Answer: Please ensure that your GCC compiler is of minimum GCC 15.1.0 version, and have `binutils` updated to the latest version. If this does not fix the problem, kindly open an issue.
|
| 200 |
+
|
| 201 |
+
4. Failing to install the `sentencepiece` package using GCC 15+
|
| 202 |
+
|
| 203 |
+
Answer: The `sentencepiece` team are aware of this as seen in [this issue](https://github.com/google/sentencepiece/issues/1108).
|
| 204 |
+
|
| 205 |
+
As a temporary workaround, please run the installation command with the following environment variables.
|
| 206 |
+
|
| 207 |
+
```bash
|
| 208 |
+
export CXXFLAGS="-include cstdint"
|
| 209 |
+
```
|
| 210 |
+
|
| 211 |
+
For example,
|
| 212 |
+
|
| 213 |
+
```bash
|
| 214 |
+
CXXFLAGS="-include cstdint" pip3 install -r requirements.txt
|
| 215 |
+
```
|
| 216 |
+
|
| 217 |
+
## Getting Help on IBM Z & LinuxONE
|
| 218 |
+
|
| 219 |
+
1. **Bugs, Feature Requests**
|
| 220 |
+
|
| 221 |
+
Please file an issue in llama.cpp and ensure that the title contains "s390x".
|
| 222 |
+
|
| 223 |
+
2. **Other Questions**
|
| 224 |
+
|
| 225 |
+
Please reach out directly to [aionz@us.ibm.com](mailto:aionz@us.ibm.com).
|
| 226 |
+
|
| 227 |
+
## Appendix A: Hardware Support Matrix
|
| 228 |
+
|
| 229 |
+
| | Support | Minimum Compiler Version |
|
| 230 |
+
| -------- | ------- | ------------------------ |
|
| 231 |
+
| IBM z15 | ✅ | |
|
| 232 |
+
| IBM z16 | ✅ | |
|
| 233 |
+
| IBM z17 | ✅ | GCC 15.1.0 |
|
| 234 |
+
| IBM zDNN | ✅ | |
|
| 235 |
+
|
| 236 |
+
- ✅ - supported and verified to run as intended
|
| 237 |
+
- 🚫 - unsupported, we are unlikely able to provide support
|
| 238 |
+
|
| 239 |
+
## Appendix B: SIMD Support Matrix
|
| 240 |
+
|
| 241 |
+
| | VX/VXE/VXE2 | zDNN | Spyre |
|
| 242 |
+
|------------|-------------|------|-------|
|
| 243 |
+
| FP32 | ✅ | ✅ | ❓ |
|
| 244 |
+
| FP16 | ✅ | ✅ | ❓ |
|
| 245 |
+
| BF16 | ✅ | ✅ | ❓ |
|
| 246 |
+
| Q4_0 | ✅ | ❓ | ❓ |
|
| 247 |
+
| Q4_1 | ✅ | ❓ | ❓ |
|
| 248 |
+
| MXFP4 | ✅ | ❓ | ❓ |
|
| 249 |
+
| Q5_0 | ✅ | ❓ | ❓ |
|
| 250 |
+
| Q5_1 | ✅ | ❓ | ❓ |
|
| 251 |
+
| Q8_0 | ✅ | ❓ | ❓ |
|
| 252 |
+
| Q2_K | 🚫 | ❓ | ❓ |
|
| 253 |
+
| Q3_K | ✅ | ❓ | ❓ |
|
| 254 |
+
| Q4_K | ✅ | ❓ | ❓ |
|
| 255 |
+
| Q5_K | ✅ | ❓ | ❓ |
|
| 256 |
+
| Q6_K | ✅ | ❓ | ❓ |
|
| 257 |
+
| TQ1_0 | 🚫 | ❓ | ❓ |
|
| 258 |
+
| TQ2_0 | 🚫 | ❓ | ❓ |
|
| 259 |
+
| IQ2_XXS | 🚫 | ❓ | ❓ |
|
| 260 |
+
| IQ2_XS | 🚫 | ❓ | ❓ |
|
| 261 |
+
| IQ2_S | 🚫 | ❓ | ❓ |
|
| 262 |
+
| IQ3_XXS | 🚫 | ❓ | ❓ |
|
| 263 |
+
| IQ3_S | 🚫 | ❓ | ❓ |
|
| 264 |
+
| IQ1_S | 🚫 | ❓ | ❓ |
|
| 265 |
+
| IQ1_M | 🚫 | ❓ | ❓ |
|
| 266 |
+
| IQ4_NL | ✅ | ❓ | ❓ |
|
| 267 |
+
| IQ4_XS | ✅ | ❓ | ❓ |
|
| 268 |
+
| FP32->FP16 | 🚫 | ❓ | ❓ |
|
| 269 |
+
| FP16->FP32 | 🚫 | ❓ | ❓ |
|
| 270 |
+
|
| 271 |
+
- ✅ - acceleration available
|
| 272 |
+
- 🚫 - acceleration unavailable, will still run using scalar implementation
|
| 273 |
+
- ❓ - acceleration unknown, please contribute if you can test it yourself
|
| 274 |
+
|
| 275 |
+
Last Updated by **Aaron Teo (aaron.teo1@ibm.com)** on Feb 15, 2026.
|
docs/build.md
ADDED
|
@@ -0,0 +1,844 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Build llama.cpp locally
|
| 2 |
+
|
| 3 |
+
The main product of this project is the `llama` library. Its C-style interface can be found in [include/llama.h](../include/llama.h).
|
| 4 |
+
|
| 5 |
+
The project also includes many example programs and tools using the `llama` library. The examples range from simple, minimal code snippets to sophisticated sub-projects such as an OpenAI-compatible HTTP server.
|
| 6 |
+
|
| 7 |
+
**To get the Code:**
|
| 8 |
+
|
| 9 |
+
```bash
|
| 10 |
+
git clone https://github.com/ggml-org/llama.cpp
|
| 11 |
+
cd llama.cpp
|
| 12 |
+
```
|
| 13 |
+
|
| 14 |
+
The following sections describe how to build with different backends and options.
|
| 15 |
+
|
| 16 |
+
* [CPU Build](#cpu-build)
|
| 17 |
+
* [BLAS Build](#blas-build)
|
| 18 |
+
* [Metal Build](#metal-build)
|
| 19 |
+
* [SYCL](#sycl)
|
| 20 |
+
* [CUDA](#cuda)
|
| 21 |
+
* [MUSA](#musa)
|
| 22 |
+
* [HIP](#hip)
|
| 23 |
+
* [Vulkan](#vulkan)
|
| 24 |
+
* [CANN](#cann)
|
| 25 |
+
* [ZenDNN](#zendnn)
|
| 26 |
+
* [Arm® KleidiAI™](#arm-kleidiai)
|
| 27 |
+
* [OpenCL](#opencl)
|
| 28 |
+
* [Android](#android-1)
|
| 29 |
+
* [OpenVINO](#openvino)
|
| 30 |
+
* [Hexagon](#hexagon)
|
| 31 |
+
* [Notes about GPU-accelerated backends](#notes-about-gpu-accelerated-backends)
|
| 32 |
+
|
| 33 |
+
## CPU Build
|
| 34 |
+
|
| 35 |
+
Build llama.cpp using `CMake`:
|
| 36 |
+
|
| 37 |
+
```bash
|
| 38 |
+
cmake -B build
|
| 39 |
+
cmake --build build --config Release
|
| 40 |
+
```
|
| 41 |
+
|
| 42 |
+
**Notes**:
|
| 43 |
+
|
| 44 |
+
- For faster compilation, add the `-j` argument to run multiple jobs in parallel, or use a generator that does this automatically such as Ninja. For example, `cmake --build build --config Release -j 8` will run 8 jobs in parallel.
|
| 45 |
+
- For faster repeated compilation, install [ccache](https://ccache.dev/)
|
| 46 |
+
- For debug builds, there are two cases:
|
| 47 |
+
|
| 48 |
+
1. Single-config generators (e.g. default = `Unix Makefiles`; note that they just ignore the `--config` flag):
|
| 49 |
+
|
| 50 |
+
```bash
|
| 51 |
+
cmake -B build -DCMAKE_BUILD_TYPE=Debug
|
| 52 |
+
cmake --build build
|
| 53 |
+
```
|
| 54 |
+
|
| 55 |
+
2. Multi-config generators (`-G` param set to Visual Studio, XCode...):
|
| 56 |
+
|
| 57 |
+
```bash
|
| 58 |
+
cmake -B build -G "Xcode"
|
| 59 |
+
cmake --build build --config Debug
|
| 60 |
+
```
|
| 61 |
+
|
| 62 |
+
For more details and a list of supported generators, see the [CMake documentation](https://cmake.org/cmake/help/latest/manual/cmake-generators.7.html).
|
| 63 |
+
- For static builds, add `-DBUILD_SHARED_LIBS=OFF`:
|
| 64 |
+
```
|
| 65 |
+
cmake -B build -DBUILD_SHARED_LIBS=OFF
|
| 66 |
+
cmake --build build --config Release
|
| 67 |
+
```
|
| 68 |
+
|
| 69 |
+
- Building for Windows (x86, x64 and arm64) with MSVC or clang as compilers:
|
| 70 |
+
- Install Visual Studio 2022, e.g. via the [Community Edition](https://visualstudio.microsoft.com/vs/community/). In the installer, select at least the following options (this also automatically installs the required additional tools like CMake,...):
|
| 71 |
+
- Tab Workload: Desktop-development with C++
|
| 72 |
+
- Tab Components (select quickly via search): C++-_CMake_ Tools for Windows, _Git_ for Windows, C++-_Clang_ Compiler for Windows, MS-Build Support for LLVM-Toolset (clang)
|
| 73 |
+
- Please remember to always use a Developer Command Prompt / PowerShell for VS2022 for git, build, test
|
| 74 |
+
- For Windows on ARM (arm64, WoA), build with:
|
| 75 |
+
```bash
|
| 76 |
+
cmake --preset arm64-windows-llvm-release -D GGML_OPENMP_FETCH=ON
|
| 77 |
+
cmake --build build-arm64-windows-llvm-release
|
| 78 |
+
```
|
| 79 |
+
- Use `ARM64 Native Tools Command Prompt for VS 2022` if you are building on an ARM64 machine.
|
| 80 |
+
- `GGML_OPENMP_FETCH` downloads the official LLVM OpenMP runtime and requires Clang, 7-Zip and network access during configuration. CMake selects the runtime from the target architecture, so this also works when cross-compiling for WoA from x64. The extracted header, import library, DLL and OpenMP license are placed under `build/_deps`. The build copies `libomp.dll` and `LICENSE-LLVM-OpenMP` to the runtime output directory and installs them together. Omit the option to use CMake's normal OpenMP detection, or pass `-D GGML_OPENMP=OFF` to disable OpenMP.
|
| 81 |
+
- For building with ninja generator and clang compiler as default:
|
| 82 |
+
- Set path:
|
| 83 |
+
```
|
| 84 |
+
set LIB=C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\um\x64;C:\Program Files\Microsoft Visual Studio\2022\Community\VC\Tools\MSVC\14.41.34120\lib\x64\uwp;C:\Program Files (x86)\Windows Kits\10\Lib\10.0.22621.0\ucrt\x64
|
| 85 |
+
```
|
| 86 |
+
- Run:
|
| 87 |
+
```bash
|
| 88 |
+
cmake --preset x64-windows-llvm-release
|
| 89 |
+
cmake --build build-x64-windows-llvm-release
|
| 90 |
+
```
|
| 91 |
+
- If you want HTTPS/TLS features, you may install OpenSSL development libraries. If not installed, the project will build and run without SSL support.
|
| 92 |
+
- **Debian / Ubuntu:** `sudo apt-get install libssl-dev`
|
| 93 |
+
- **Fedora / RHEL / Rocky / Alma:** `sudo dnf install openssl-devel`
|
| 94 |
+
- **Arch / Manjaro:** `sudo pacman -S openssl`
|
| 95 |
+
|
| 96 |
+
## BLAS Build
|
| 97 |
+
|
| 98 |
+
Building the program with BLAS support may lead to some performance improvements in prompt processing using batch sizes higher than 32 (the default is 512). Using BLAS doesn't affect the generation performance. There are currently several different BLAS implementations available for build and use:
|
| 99 |
+
|
| 100 |
+
### Accelerate Framework
|
| 101 |
+
|
| 102 |
+
This is only available on Mac PCs and it's enabled by default. You can just build using the normal instructions.
|
| 103 |
+
|
| 104 |
+
### OpenBLAS
|
| 105 |
+
|
| 106 |
+
This provides BLAS acceleration using only the CPU. Make sure to have OpenBLAS installed on your machine.
|
| 107 |
+
|
| 108 |
+
- Using `CMake` on Linux:
|
| 109 |
+
|
| 110 |
+
```bash
|
| 111 |
+
cmake -B build -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=OpenBLAS
|
| 112 |
+
cmake --build build --config Release
|
| 113 |
+
```
|
| 114 |
+
|
| 115 |
+
### BLIS
|
| 116 |
+
|
| 117 |
+
Check [BLIS.md](./backend/BLIS.md) for more information.
|
| 118 |
+
|
| 119 |
+
### Intel oneMKL
|
| 120 |
+
|
| 121 |
+
Building through oneAPI compilers will make avx_vnni instruction set available for intel processors that do not support avx512 and avx512_vnni. Please note that this build config **does not support Intel GPU**. For Intel GPU support, please refer to [llama.cpp for SYCL](./backend/SYCL.md).
|
| 122 |
+
|
| 123 |
+
- Using manual oneAPI installation:
|
| 124 |
+
By default, `GGML_BLAS_VENDOR` is set to `Generic`, so if you already sourced intel environment script and assign `-DGGML_BLAS=ON` in cmake, the mkl version of Blas will automatically been selected. Otherwise please install oneAPI and follow the below steps:
|
| 125 |
+
```bash
|
| 126 |
+
source /opt/intel/oneapi/setvars.sh # You can skip this step if in oneapi-basekit docker image, only required for manual installation
|
| 127 |
+
cmake -B build -DGGML_BLAS=ON -DGGML_BLAS_VENDOR=Intel10_64lp -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_NATIVE=ON
|
| 128 |
+
cmake --build build --config Release
|
| 129 |
+
```
|
| 130 |
+
|
| 131 |
+
- Using oneAPI docker image:
|
| 132 |
+
If you do not want to source the environment vars and install oneAPI manually, you can also build the code using intel docker container: [oneAPI-basekit](https://hub.docker.com/r/intel/oneapi-basekit). Then, you can use the commands given above.
|
| 133 |
+
|
| 134 |
+
Check [Optimizing and Running LLaMA2 on Intel® CPU](https://builders.intel.com/solutionslibrary/optimizing-and-running-llama2-on-intel-cpu) for more information.
|
| 135 |
+
|
| 136 |
+
### Other BLAS libraries
|
| 137 |
+
|
| 138 |
+
Any other BLAS library can be used by setting the `GGML_BLAS_VENDOR` option. See the [CMake documentation](https://cmake.org/cmake/help/latest/module/FindBLAS.html#blas-lapack-vendors) for a list of supported vendors.
|
| 139 |
+
|
| 140 |
+
## Metal Build
|
| 141 |
+
|
| 142 |
+
On MacOS, Metal is enabled by default. Using Metal makes the computation run on the GPU.
|
| 143 |
+
To disable the Metal build at compile time use the `-DGGML_METAL=OFF` cmake option.
|
| 144 |
+
|
| 145 |
+
When built with Metal support, you can explicitly disable GPU inference with the `--n-gpu-layers 0` command-line argument.
|
| 146 |
+
|
| 147 |
+
## SYCL
|
| 148 |
+
|
| 149 |
+
SYCL is a higher-level programming model to improve programming productivity on various hardware accelerators.
|
| 150 |
+
|
| 151 |
+
llama.cpp based on SYCL is used to **support Intel GPU** (Data Center Max series, Flex series, Arc series, Built-in GPU and iGPU).
|
| 152 |
+
|
| 153 |
+
For detailed info, please refer to [llama.cpp for SYCL](./backend/SYCL.md).
|
| 154 |
+
|
| 155 |
+
## CUDA
|
| 156 |
+
|
| 157 |
+
This provides GPU acceleration using an NVIDIA GPU. Make sure to have the [CUDA toolkit](https://developer.nvidia.com/cuda-toolkit) installed.
|
| 158 |
+
|
| 159 |
+
#### Download directly from NVIDIA
|
| 160 |
+
You may find the official downloads here: [NVIDIA developer site](https://developer.nvidia.com/cuda-downloads).
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
#### Compile and run inside a Fedora Toolbox Container
|
| 164 |
+
We also have a [guide](./backend/CUDA-FEDORA.md) for setting up CUDA toolkit in a Fedora [toolbox container](https://containertoolbx.org/).
|
| 165 |
+
|
| 166 |
+
**Recommended for:**
|
| 167 |
+
- ***Necessary*** for users of [Atomic Desktops for Fedora](https://fedoraproject.org/atomic-desktops/); such as: [Silverblue](https://fedoraproject.org/atomic-desktops/silverblue/) and [Kinoite](https://fedoraproject.org/atomic-desktops/kinoite/).
|
| 168 |
+
- (there are no supported CUDA packages for these systems)
|
| 169 |
+
- ***Necessary*** for users that have a host that is not a: [Supported Nvidia CUDA Release Platform](https://developer.nvidia.com/cuda-downloads).
|
| 170 |
+
- (for example, you may have [Fedora 42 Beta](https://fedoramagazine.org/announcing-fedora-linux-42-beta/) as your host operating system)
|
| 171 |
+
- ***Convenient*** For those running [Fedora Workstation](https://fedoraproject.org/workstation/) or [Fedora KDE Plasma Desktop](https://fedoraproject.org/spins/kde), and want to keep their host system clean.
|
| 172 |
+
- *Optionally* toolbox packages are available: [Arch Linux](https://archlinux.org/), [Red Hat Enterprise Linux >= 8.5](https://www.redhat.com/en/technologies/linux-platforms/enterprise-linux), or [Ubuntu](https://ubuntu.com/download)
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
### Compilation
|
| 176 |
+
|
| 177 |
+
Make sure to read the notes about the CPU build for general instructions for e.g. speeding up the compilation.
|
| 178 |
+
|
| 179 |
+
```bash
|
| 180 |
+
cmake -B build -DGGML_CUDA=ON
|
| 181 |
+
cmake --build build --config Release
|
| 182 |
+
```
|
| 183 |
+
|
| 184 |
+
### Non-Native Builds
|
| 185 |
+
|
| 186 |
+
By default llama.cpp will be built for the hardware that is connected to the system at that time.
|
| 187 |
+
For a build covering all CUDA GPUs, disable `GGML_NATIVE`:
|
| 188 |
+
|
| 189 |
+
```bash
|
| 190 |
+
cmake -B build -DGGML_CUDA=ON -DGGML_NATIVE=OFF
|
| 191 |
+
```
|
| 192 |
+
|
| 193 |
+
The resulting binary should run on all CUDA GPUs with optimal performance, though some just-in-time compilation may be required.
|
| 194 |
+
|
| 195 |
+
### Override Compute Capability Specifications
|
| 196 |
+
|
| 197 |
+
If `nvcc` cannot detect your gpu, you may get compile warnings such as:
|
| 198 |
+
```text
|
| 199 |
+
nvcc warning : Cannot find valid GPU for '-arch=native', default arch is used
|
| 200 |
+
```
|
| 201 |
+
|
| 202 |
+
One option is to do a non-native build as described above.
|
| 203 |
+
However, this will result in a large binary that takes a long time to compile.
|
| 204 |
+
Alternatively it is also possible to explicitly specify CUDA architectures.
|
| 205 |
+
This may also make sense for a non-native build, for that one should look at the logic in `ggml/src/ggml-cuda/CMakeLists.txt` as a starting point.
|
| 206 |
+
|
| 207 |
+
To override the default CUDA architectures:
|
| 208 |
+
|
| 209 |
+
#### 1. Take note of the `Compute Capability` of your NVIDIA devices: ["CUDA: Your GPU Compute > Capability"](https://developer.nvidia.com/cuda-gpus).
|
| 210 |
+
|
| 211 |
+
```text
|
| 212 |
+
GeForce RTX 4090 8.9
|
| 213 |
+
GeForce RTX 3080 Ti 8.6
|
| 214 |
+
GeForce RTX 3070 8.6
|
| 215 |
+
```
|
| 216 |
+
|
| 217 |
+
#### 2. Manually list each varying `Compute Capability` in the `CMAKE_CUDA_ARCHITECTURES` list.
|
| 218 |
+
|
| 219 |
+
```bash
|
| 220 |
+
cmake -B build -DGGML_CUDA=ON -DCMAKE_CUDA_ARCHITECTURES="86;89"
|
| 221 |
+
```
|
| 222 |
+
|
| 223 |
+
### Overriding the CUDA Version
|
| 224 |
+
|
| 225 |
+
If you have multiple CUDA installations on your system and want to compile llama.cpp for a specific one, e.g. for CUDA 11.7 installed under `/opt/cuda-11.7`:
|
| 226 |
+
|
| 227 |
+
```bash
|
| 228 |
+
cmake -B build -DGGML_CUDA=ON -DCMAKE_CUDA_COMPILER=/opt/cuda-11.7/bin/nvcc -DCMAKE_INSTALL_RPATH="/opt/cuda-11.7/lib64;\$ORIGIN" -DCMAKE_BUILD_WITH_INSTALL_RPATH=ON
|
| 229 |
+
```
|
| 230 |
+
|
| 231 |
+
#### Fixing Compatibility Issues with Old CUDA and New glibc
|
| 232 |
+
|
| 233 |
+
If you try to use an old CUDA version (e.g. v11.7) with a new glibc version you can get errors like this:
|
| 234 |
+
|
| 235 |
+
```
|
| 236 |
+
/usr/include/bits/mathcalls.h(83): error: exception specification is
|
| 237 |
+
incompatible with that of previous function "cospi"
|
| 238 |
+
|
| 239 |
+
|
| 240 |
+
/opt/cuda-11.7/bin/../targets/x86_64-linux/include/crt/math_functions.h(5545):
|
| 241 |
+
here
|
| 242 |
+
```
|
| 243 |
+
|
| 244 |
+
It seems the least bad solution is to patch the CUDA installation to declare the correct signatures.
|
| 245 |
+
Replace the following lines in `/path/to/your/cuda/installation/targets/x86_64-linux/include/crt/math_functions.h`:
|
| 246 |
+
|
| 247 |
+
```C++
|
| 248 |
+
// original lines
|
| 249 |
+
extern __DEVICE_FUNCTIONS_DECL__ __device_builtin__ double cospi(double x);
|
| 250 |
+
extern __DEVICE_FUNCTIONS_DECL__ __device_builtin__ float cospif(float x);
|
| 251 |
+
extern __DEVICE_FUNCTIONS_DECL__ __device_builtin__ double sinpi(double x);
|
| 252 |
+
extern __DEVICE_FUNCTIONS_DECL__ __device_builtin__ float sinpif(float x);
|
| 253 |
+
extern __DEVICE_FUNCTIONS_DECL__ __device_builtin__ double rsqrt(double x);
|
| 254 |
+
extern __DEVICE_FUNCTIONS_DECL__ __device_builtin__ float rsqrtf(float x);
|
| 255 |
+
|
| 256 |
+
// edited lines
|
| 257 |
+
extern __DEVICE_FUNCTIONS_DECL__ __device_builtin__ double cospi(double x) noexcept (true);
|
| 258 |
+
extern __DEVICE_FUNCTIONS_DECL__ __device_builtin__ float cospif(float x) noexcept (true);
|
| 259 |
+
extern __DEVICE_FUNCTIONS_DECL__ __device_builtin__ double sinpi(double x) noexcept (true);
|
| 260 |
+
extern __DEVICE_FUNCTIONS_DECL__ __device_builtin__ float sinpif(float x) noexcept (true);
|
| 261 |
+
extern __DEVICE_FUNCTIONS_DECL__ __device_builtin__ double rsqrt(double x) noexcept (true);
|
| 262 |
+
extern __DEVICE_FUNCTIONS_DECL__ __device_builtin__ float rsqrtf(float x) noexcept (true);
|
| 263 |
+
```
|
| 264 |
+
|
| 265 |
+
### Runtime CUDA environmental variables
|
| 266 |
+
|
| 267 |
+
You may set the [cuda environmental variables](https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#env-vars) at runtime.
|
| 268 |
+
|
| 269 |
+
```bash
|
| 270 |
+
# Use `CUDA_VISIBLE_DEVICES` to hide the first compute device.
|
| 271 |
+
CUDA_VISIBLE_DEVICES="-0" ./build/bin/llama-server --model /srv/models/llama.gguf
|
| 272 |
+
```
|
| 273 |
+
|
| 274 |
+
#### CUDA_SCALE_LAUNCH_QUEUES
|
| 275 |
+
|
| 276 |
+
The environment variable [`CUDA_SCALE_LAUNCH_QUEUES`](https://docs.nvidia.com/cuda/cuda-programming-guide/05-appendices/environment-variables.html#cuda-scale-launch-queues) controls the size of CUDA's command buffer, which determines how many GPU operations can be queued before the CPU must wait for the GPU to catch up. A larger buffer reduces CPU-side stalls and allows more work to be queued on a GPU.
|
| 277 |
+
|
| 278 |
+
Consider setting `CUDA_SCALE_LAUNCH_QUEUES=4x`, which increases the CUDA command buffer to 4 times its default size. This optimization is particularly beneficial for **Multi-GPU setups with pipeline parallelism**, where it significantly improves prompt processing throughput by allowing more operations to be enqueued across GPUs.
|
| 279 |
+
|
| 280 |
+
#### GGML_CUDA_CUBLAS_COMPUTE_TYPE
|
| 281 |
+
|
| 282 |
+
Override default, speed-optimized compute types for cuBLAS matrix multiplications.
|
| 283 |
+
Legal values: `auto`, `f16`, `fp16`, `bf16`, `f32`, `fp32`.
|
| 284 |
+
|
| 285 |
+
### Unified Memory
|
| 286 |
+
|
| 287 |
+
The environment variable `GGML_CUDA_ENABLE_UNIFIED_MEMORY=1` can be used to enable unified memory in Linux. This allows swapping to system RAM instead of crashing when the GPU VRAM is exhausted. In Windows this setting is available in the NVIDIA control panel as `System Memory Fallback`.
|
| 288 |
+
|
| 289 |
+
### Peer Access
|
| 290 |
+
|
| 291 |
+
The environment variable `GGML_CUDA_P2P` can be set to enable peer-to-peer access between multiple GPUs, allowing them to transfer data directly rather than to go through system memory.
|
| 292 |
+
Requires driver support (usually restricted to workstation/datacenter GPUs).
|
| 293 |
+
May cause crashes or corrupted outputs for some motherboards and BIOS settings (e.g. IOMMU).
|
| 294 |
+
|
| 295 |
+
### Performance Tuning
|
| 296 |
+
|
| 297 |
+
The following compilation options are also available to tweak performance:
|
| 298 |
+
|
| 299 |
+
| Option | Legal values | Default | Description |
|
| 300 |
+
|-------------------------------|------------------------|---------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
| 301 |
+
| GGML_CUDA_FORCE_MMQ | Boolean | false | Force the use of custom matrix multiplication kernels for quantized models instead of FP16 cuBLAS even if there is no int8 tensor core implementation available (affects V100, CDNA and RDNA3+). MMQ kernels are enabled by default on GPUs with int8 tensor core support. With MMQ force enabled, speed for large batch sizes will be worse but VRAM consumption will be lower. |
|
| 302 |
+
| GGML_CUDA_FORCE_CUBLAS | Boolean | false | Force the use of FP16 cuBLAS instead of custom matrix multiplication kernels for quantized models. There may be issues with numerical overflows (except for V100, CDNA and RDNA4 which use FP32 compute type by default) and memory use will be higher. Prompt processing may become faster on recent datacenter GPUs (the custom kernels were tuned primarily for RTX 3000/4000). |
|
| 303 |
+
| GGML_CUDA_FA_ALL_QUANTS | Boolean | false | Compile support for all KV cache quantization type (combinations) for the FlashAttention CUDA kernels. More fine-grained control over KV cache size but compilation takes much longer. |
|
| 304 |
+
|
| 305 |
+
## MUSA
|
| 306 |
+
|
| 307 |
+
This provides GPU acceleration using a Moore Threads GPU. Make sure to have the [MUSA SDK](https://developer.mthreads.com/musa/musa-sdk) installed.
|
| 308 |
+
|
| 309 |
+
#### Download directly from Moore Threads
|
| 310 |
+
|
| 311 |
+
You may find the official downloads here: [Moore Threads developer site](https://developer.mthreads.com/sdk/download/musa).
|
| 312 |
+
|
| 313 |
+
### Compilation
|
| 314 |
+
|
| 315 |
+
```bash
|
| 316 |
+
cmake -B build -DGGML_MUSA=ON
|
| 317 |
+
cmake --build build --config Release
|
| 318 |
+
```
|
| 319 |
+
|
| 320 |
+
#### Override Compute Capability Specifications
|
| 321 |
+
|
| 322 |
+
By default, all supported compute capabilities are enabled. To customize this behavior, you can specify the `MUSA_ARCHITECTURES` option in the CMake command:
|
| 323 |
+
|
| 324 |
+
```bash
|
| 325 |
+
cmake -B build -DGGML_MUSA=ON -DMUSA_ARCHITECTURES="21"
|
| 326 |
+
cmake --build build --config Release
|
| 327 |
+
```
|
| 328 |
+
|
| 329 |
+
This configuration enables only compute capability `2.1` (MTT S80) during compilation, which can help reduce compilation time.
|
| 330 |
+
|
| 331 |
+
#### Compilation options
|
| 332 |
+
|
| 333 |
+
Most of the compilation options available for CUDA should also be available for MUSA, though they haven't been thoroughly tested yet.
|
| 334 |
+
|
| 335 |
+
- For static builds, add `-DBUILD_SHARED_LIBS=OFF` and `-DCMAKE_POSITION_INDEPENDENT_CODE=ON`:
|
| 336 |
+
```
|
| 337 |
+
cmake -B build -DGGML_MUSA=ON \
|
| 338 |
+
-DBUILD_SHARED_LIBS=OFF -DCMAKE_POSITION_INDEPENDENT_CODE=ON
|
| 339 |
+
cmake --build build --config Release
|
| 340 |
+
```
|
| 341 |
+
|
| 342 |
+
### Runtime MUSA environmental variables
|
| 343 |
+
|
| 344 |
+
You may set the [musa environmental variables](https://docs.mthreads.com/musa-sdk/musa-sdk-doc-online/programming_guide/Z%E9%99%84%E5%BD%95/) at runtime.
|
| 345 |
+
|
| 346 |
+
```bash
|
| 347 |
+
# Use `MUSA_VISIBLE_DEVICES` to hide the first compute device.
|
| 348 |
+
MUSA_VISIBLE_DEVICES="-0" ./build/bin/llama-server --model /srv/models/llama.gguf
|
| 349 |
+
```
|
| 350 |
+
|
| 351 |
+
### Unified Memory
|
| 352 |
+
|
| 353 |
+
The environment variable `GGML_CUDA_ENABLE_UNIFIED_MEMORY=1` can be used to enable unified memory in Linux. This allows swapping to system RAM instead of crashing when the GPU VRAM is exhausted.
|
| 354 |
+
|
| 355 |
+
## HIP
|
| 356 |
+
|
| 357 |
+
This provides GPU acceleration on HIP-supported AMD GPUs.
|
| 358 |
+
Make sure to have ROCm installed.
|
| 359 |
+
You can download it from your Linux distro's package manager or from here: [ROCm Quick Start (Linux)](https://rocm.docs.amd.com/projects/install-on-linux/en/latest/tutorial/quick-start.html#rocm-install-quick).
|
| 360 |
+
|
| 361 |
+
- Using `CMake` for Linux (assuming a gfx1030-compatible AMD GPU):
|
| 362 |
+
```bash
|
| 363 |
+
HIPCXX="$(hipconfig -l)/clang" HIP_PATH="$(hipconfig -R)" \
|
| 364 |
+
cmake -S . -B build -DGGML_HIP=ON -DGPU_TARGETS=gfx1030 -DCMAKE_BUILD_TYPE=Release \
|
| 365 |
+
&& cmake --build build --config Release -- -j 16
|
| 366 |
+
```
|
| 367 |
+
|
| 368 |
+
Note: `GPU_TARGETS` is optional, omitting it will build the code for all GPUs in the current system.
|
| 369 |
+
|
| 370 |
+
Note that if you get the following error:
|
| 371 |
+
```
|
| 372 |
+
clang: error: cannot find ROCm device library; provide its path via '--rocm-path' or '--rocm-device-lib-path', or pass '-nogpulib' to build without ROCm device library
|
| 373 |
+
```
|
| 374 |
+
Try searching for a directory under `HIP_PATH` that contains the file
|
| 375 |
+
`oclc_abi_version_400.bc`. Then, add the following to the start of the
|
| 376 |
+
command: `HIP_DEVICE_LIB_PATH=<directory-you-just-found>`, so something
|
| 377 |
+
like:
|
| 378 |
+
```bash
|
| 379 |
+
HIPCXX="$(hipconfig -l)/clang" HIP_PATH="$(hipconfig -p)" \
|
| 380 |
+
HIP_DEVICE_LIB_PATH=<directory-you-just-found> \
|
| 381 |
+
cmake -S . -B build -DGGML_HIP=ON -DGPU_TARGETS=gfx1030 -DCMAKE_BUILD_TYPE=Release \
|
| 382 |
+
&& cmake --build build -- -j 16
|
| 383 |
+
```
|
| 384 |
+
|
| 385 |
+
- Using `CMake` for Windows (using x64 Native Tools Command Prompt for VS, and assuming a gfx1100-compatible AMD GPU):
|
| 386 |
+
```bash
|
| 387 |
+
set PATH=%HIP_PATH%\bin;%PATH%
|
| 388 |
+
cmake -S . -B build -G Ninja -DGPU_TARGETS=gfx1100 -DGGML_HIP=ON -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DCMAKE_BUILD_TYPE=Release
|
| 389 |
+
cmake --build build
|
| 390 |
+
```
|
| 391 |
+
If necessary, adapt `GPU_TARGETS` to the GPU arch you want to compile for. The above example uses `gfx1100` that corresponds to Radeon RX 7900XTX/XT/GRE. You can find a list of targets [here](https://llvm.org/docs/AMDGPUUsage.html#processors)
|
| 392 |
+
Find your gpu version string by matching the most significant version information from `rocminfo | grep gfx | head -1 | awk '{print $2}'` with the list of processors, e.g. `gfx1035` maps to `gfx1030`.
|
| 393 |
+
|
| 394 |
+
|
| 395 |
+
The environment variable [`HIP_VISIBLE_DEVICES`](https://rocm.docs.amd.com/en/latest/understand/gpu_isolation.html#hip-visible-devices) can be used to specify which GPU(s) will be used.
|
| 396 |
+
If your GPU is not officially supported you can use the environment variable [`HSA_OVERRIDE_GFX_VERSION`] set to a similar GPU, for example 10.3.0 on RDNA2 (e.g. gfx1030, gfx1031, or gfx1035) or 11.0.0 on RDNA3. Note that [`HSA_OVERRIDE_GFX_VERSION`] is [not supported on Windows](https://github.com/ROCm/ROCm/issues/2654)
|
| 397 |
+
|
| 398 |
+
### Unified Memory
|
| 399 |
+
|
| 400 |
+
On Linux it is possible to use unified memory architecture (UMA) to share main memory between the CPU and integrated GPU by setting environment variable `GGML_CUDA_ENABLE_UNIFIED_MEMORY=1`. However, this hurts performance for non-integrated GPUs (but enables working with integrated GPUs).
|
| 401 |
+
|
| 402 |
+
## Vulkan
|
| 403 |
+
|
| 404 |
+
### For Windows Users:
|
| 405 |
+
**w64devkit**
|
| 406 |
+
|
| 407 |
+
Download and extract [`w64devkit`](https://github.com/skeeto/w64devkit/releases).
|
| 408 |
+
|
| 409 |
+
Download and install the [`Vulkan SDK`](https://vulkan.lunarg.com/sdk/home#windows) with the default settings.
|
| 410 |
+
|
| 411 |
+
Launch `w64devkit.exe` and run the following commands to copy Vulkan dependencies:
|
| 412 |
+
```sh
|
| 413 |
+
SDK_VERSION=1.3.283.0
|
| 414 |
+
cp /VulkanSDK/$SDK_VERSION/Bin/glslc.exe $W64DEVKIT_HOME/bin/
|
| 415 |
+
cp /VulkanSDK/$SDK_VERSION/Lib/vulkan-1.lib $W64DEVKIT_HOME/x86_64-w64-mingw32/lib/
|
| 416 |
+
cp -r /VulkanSDK/$SDK_VERSION/Include/* $W64DEVKIT_HOME/x86_64-w64-mingw32/include/
|
| 417 |
+
cat > $W64DEVKIT_HOME/x86_64-w64-mingw32/lib/pkgconfig/vulkan.pc <<EOF
|
| 418 |
+
Name: Vulkan-Loader
|
| 419 |
+
Description: Vulkan Loader
|
| 420 |
+
Version: $SDK_VERSION
|
| 421 |
+
Libs: -lvulkan-1
|
| 422 |
+
EOF
|
| 423 |
+
|
| 424 |
+
```
|
| 425 |
+
|
| 426 |
+
Switch into the `llama.cpp` directory and build using CMake.
|
| 427 |
+
```sh
|
| 428 |
+
cmake -B build -DGGML_VULKAN=ON
|
| 429 |
+
cmake --build build --config Release
|
| 430 |
+
```
|
| 431 |
+
|
| 432 |
+
**Git Bash MINGW64**
|
| 433 |
+
|
| 434 |
+
Download and install [`Git-SCM`](https://git-scm.com/downloads/win) with the default settings
|
| 435 |
+
|
| 436 |
+
Download and install [`Visual Studio Community Edition`](https://visualstudio.microsoft.com/) and make sure you select `C++`
|
| 437 |
+
|
| 438 |
+
Download and install [`CMake`](https://cmake.org/download/) with the default settings
|
| 439 |
+
|
| 440 |
+
Download and install the [`Vulkan SDK`](https://vulkan.lunarg.com/sdk/home#windows) with the default settings.
|
| 441 |
+
|
| 442 |
+
Go into your `llama.cpp` directory and right click, select `Open Git Bash Here` and then run the following commands
|
| 443 |
+
|
| 444 |
+
```
|
| 445 |
+
cmake -B build -DGGML_VULKAN=ON
|
| 446 |
+
cmake --build build --config Release
|
| 447 |
+
```
|
| 448 |
+
|
| 449 |
+
Now you can load the model in conversation mode using `Vulkan`
|
| 450 |
+
|
| 451 |
+
```sh
|
| 452 |
+
build/bin/Release/llama-cli -m "[PATH TO MODEL]" -ngl 100 -c 16384 -t 10 -n -2 -cnv
|
| 453 |
+
```
|
| 454 |
+
|
| 455 |
+
**MSYS2**
|
| 456 |
+
|
| 457 |
+
Install [MSYS2](https://www.msys2.org/) and then run the following commands in a UCRT terminal to install dependencies.
|
| 458 |
+
```sh
|
| 459 |
+
pacman -S git \
|
| 460 |
+
mingw-w64-ucrt-x86_64-gcc \
|
| 461 |
+
mingw-w64-ucrt-x86_64-cmake \
|
| 462 |
+
mingw-w64-ucrt-x86_64-vulkan-devel \
|
| 463 |
+
mingw-w64-ucrt-x86_64-shaderc \
|
| 464 |
+
mingw-w64-ucrt-x86_64-spirv-headers
|
| 465 |
+
```
|
| 466 |
+
|
| 467 |
+
Switch into the `llama.cpp` directory and build using CMake.
|
| 468 |
+
```sh
|
| 469 |
+
cmake -B build -DGGML_VULKAN=ON
|
| 470 |
+
cmake --build build --config Release
|
| 471 |
+
```
|
| 472 |
+
|
| 473 |
+
### For Docker users:
|
| 474 |
+
|
| 475 |
+
You don't need to install the Vulkan SDK. It will be installed inside the container.
|
| 476 |
+
|
| 477 |
+
```sh
|
| 478 |
+
# Build the image
|
| 479 |
+
docker build -t llama-cpp-vulkan --target light -f .devops/vulkan.Dockerfile .
|
| 480 |
+
|
| 481 |
+
# Then, use it:
|
| 482 |
+
docker run -it --rm -v "$(pwd):/app:Z" --device /dev/dri/renderD128:/dev/dri/renderD128 --device /dev/dri/card1:/dev/dri/card1 llama-cpp-vulkan -m "/app/models/YOUR_MODEL_FILE" -p "Building a website can be done in 10 simple steps:" -n 400 -e -ngl 33
|
| 483 |
+
```
|
| 484 |
+
|
| 485 |
+
### For Linux users:
|
| 486 |
+
|
| 487 |
+
#### Using the LunarG Vulkan SDK
|
| 488 |
+
|
| 489 |
+
First, follow the official LunarG instructions for the installation and setup of the Vulkan SDK in the [Getting Started with the Linux Tarball Vulkan SDK](https://vulkan.lunarg.com/doc/sdk/latest/linux/getting_started.html) guide.
|
| 490 |
+
|
| 491 |
+
> [!IMPORTANT]
|
| 492 |
+
> After completing the first step, ensure that you have used the `source` command on the `setup_env.sh` file inside of the Vulkan SDK in your current terminal session. Otherwise, the build won't work. Additionally, if you close out of your terminal, you must perform this step again if you intend to perform a build. However, there are ways to make this persistent. Refer to the Vulkan SDK guide linked in the first step for more information about any of this.
|
| 493 |
+
|
| 494 |
+
#### Using system packages
|
| 495 |
+
|
| 496 |
+
On Debian / Ubuntu, you can install the required dependencies using:
|
| 497 |
+
```sh
|
| 498 |
+
sudo apt-get install libvulkan-dev glslc spirv-headers
|
| 499 |
+
```
|
| 500 |
+
|
| 501 |
+
SPIRV-Headers (`spirv/unified1/spirv.hpp`) are required for the Vulkan backend and are **not** always pulled in by the Vulkan loader dev package alone. Other distros use names such as `spirv-headers` (Ubuntu / Debian / Arch), or `spirv-headers-devel` (Fedora / openSUSE). On Windows, the LunarG Vulkan SDK’s `Include` directory already contains these headers.
|
| 502 |
+
|
| 503 |
+
#### Common steps
|
| 504 |
+
|
| 505 |
+
Second, after verifying that you have followed all of the SDK installation/setup steps, use this command to make sure before proceeding:
|
| 506 |
+
```bash
|
| 507 |
+
vulkaninfo
|
| 508 |
+
```
|
| 509 |
+
|
| 510 |
+
Then, assuming you have `cd` into your llama.cpp folder and there are no errors with running `vulkaninfo`, you can proceed to build llama.cpp using the CMake commands below:
|
| 511 |
+
```bash
|
| 512 |
+
cmake -B build -DGGML_VULKAN=1
|
| 513 |
+
cmake --build build --config Release
|
| 514 |
+
```
|
| 515 |
+
|
| 516 |
+
Finally, after finishing your build, you should be able to do something like this:
|
| 517 |
+
```bash
|
| 518 |
+
# Test the output binary
|
| 519 |
+
# "-ngl 99" should offload all of the layers to GPU for most (if not all) models.
|
| 520 |
+
./build/bin/llama-cli -m "PATH_TO_MODEL" -p "Hi you how are you" -ngl 99
|
| 521 |
+
|
| 522 |
+
# You should see in the output, ggml_vulkan detected your GPU. For example:
|
| 523 |
+
# ggml_vulkan: Using Intel(R) Graphics (ADL GT2) | uma: 1 | fp16: 1 | warp size: 32
|
| 524 |
+
```
|
| 525 |
+
|
| 526 |
+
### For Mac users:
|
| 527 |
+
|
| 528 |
+
Generally, follow LunarG's [Getting Started with the MacOS Vulkan SDK](https://vulkan.lunarg.com/doc/sdk/latest/mac/getting_started.html) guide for installation and setup of the Vulkan SDK. There are two options of Vulkan drivers on macOS, both of which implement translation layers to map Vulkan to Metal. They can be hot-swapped by setting the `VK_ICD_FILENAMES` environment variable to point to the respective ICD JSON file.
|
| 529 |
+
|
| 530 |
+
Check the box for "KosmicKrisp" during the LunarG Vulkan SDK installation.
|
| 531 |
+
|
| 532 |
+
Set environment variable for the LunarG Vulkan SDK after installation (and optionally add to your shell profile for persistence):
|
| 533 |
+
```bash
|
| 534 |
+
source /path/to/vulkan-sdk/setup-env.sh
|
| 535 |
+
```
|
| 536 |
+
|
| 537 |
+
#### Using MoltenVK
|
| 538 |
+
|
| 539 |
+
MoltenVK is the default Vulkan driver installed with the LunarG Vulkan SDK on macOS, so you can use the above environment variable settings as is.
|
| 540 |
+
|
| 541 |
+
#### Using KosmicKrisp
|
| 542 |
+
|
| 543 |
+
Override the environment variable for KosmicKrisp:
|
| 544 |
+
```bash
|
| 545 |
+
export VK_ICD_FILENAMES=$VULKAN_SDK/share/vulkan/icd.d/libkosmickrisp_icd.json
|
| 546 |
+
export VK_DRIVER_FILES=$VULKAN_SDK/share/vulkan/icd.d/libkosmickrisp_icd.json
|
| 547 |
+
```
|
| 548 |
+
|
| 549 |
+
#### Build
|
| 550 |
+
|
| 551 |
+
This is the only step different from [above](#common-steps) instructions.
|
| 552 |
+
```bash
|
| 553 |
+
cmake -B build -DGGML_VULKAN=1 -DGGML_METAL=OFF
|
| 554 |
+
cmake --build build --config Release
|
| 555 |
+
```
|
| 556 |
+
|
| 557 |
+
## CANN
|
| 558 |
+
This provides NPU acceleration using the AI cores of your Ascend NPU. And [CANN](https://www.hiascend.com/en/software/cann) is a hierarchical APIs to help you to quickly build AI applications and service based on Ascend NPU.
|
| 559 |
+
|
| 560 |
+
For more information about Ascend NPU in [Ascend Community](https://www.hiascend.com/en/).
|
| 561 |
+
|
| 562 |
+
Make sure to have the CANN toolkit installed. You can download it from here: [CANN Toolkit](https://www.hiascend.com/developer/download/community/result?module=cann)
|
| 563 |
+
|
| 564 |
+
Go to `llama.cpp` directory and build using CMake.
|
| 565 |
+
```bash
|
| 566 |
+
cmake -B build -DGGML_CANN=on -DCMAKE_BUILD_TYPE=release
|
| 567 |
+
cmake --build build --config release
|
| 568 |
+
```
|
| 569 |
+
|
| 570 |
+
You can test with:
|
| 571 |
+
|
| 572 |
+
```bash
|
| 573 |
+
./build/bin/llama-cli -m PATH_TO_MODEL -p "Building a website can be done in 10 steps:" -ngl 32
|
| 574 |
+
```
|
| 575 |
+
|
| 576 |
+
If the following info is output on screen, you are using `llama.cpp` with the CANN backend:
|
| 577 |
+
```bash
|
| 578 |
+
llm_load_tensors: CANN model buffer size = 13313.00 MiB
|
| 579 |
+
llama_new_context_with_model: CANN compute buffer size = 1260.81 MiB
|
| 580 |
+
```
|
| 581 |
+
|
| 582 |
+
For detailed info, such as model/device supports, CANN install, please refer to [llama.cpp for CANN](./backend/CANN.md).
|
| 583 |
+
|
| 584 |
+
## ZenDNN
|
| 585 |
+
|
| 586 |
+
ZenDNN provides optimized deep learning primitives for AMD EPYC™ CPUs. It accelerates matrix multiplication operations for inference workloads.
|
| 587 |
+
|
| 588 |
+
### Compilation
|
| 589 |
+
|
| 590 |
+
- Using `CMake` on Linux (automatic build):
|
| 591 |
+
|
| 592 |
+
```bash
|
| 593 |
+
cmake -B build -DGGML_ZENDNN=ON
|
| 594 |
+
cmake --build build --config Release
|
| 595 |
+
```
|
| 596 |
+
|
| 597 |
+
The first build will automatically download and build ZenDNN, which may take 5-10 minutes. Subsequent builds will be much faster.
|
| 598 |
+
|
| 599 |
+
- Using `CMake` with custom ZenDNN installation:
|
| 600 |
+
|
| 601 |
+
```bash
|
| 602 |
+
cmake -B build -DGGML_ZENDNN=ON -DZENDNN_ROOT=/path/to/zendnn/install
|
| 603 |
+
cmake --build build --config Release
|
| 604 |
+
```
|
| 605 |
+
|
| 606 |
+
### Testing
|
| 607 |
+
|
| 608 |
+
You can test with:
|
| 609 |
+
|
| 610 |
+
```bash
|
| 611 |
+
./build/bin/llama-cli -m PATH_TO_MODEL -p "Building a website can be done in 10 steps:" -n 50
|
| 612 |
+
```
|
| 613 |
+
|
| 614 |
+
For detailed information about hardware support, setup instructions, and performance optimization, refer to [llama.cpp for ZenDNN](./backend/ZenDNN.md).
|
| 615 |
+
|
| 616 |
+
## Arm® KleidiAI™
|
| 617 |
+
KleidiAI provides optimized Arm CPU microkernels used by the ggml CPU backend. Enabling it at build time makes those kernels available; it does not force every operation to use KleidiAI. At runtime, llama.cpp selects the best compatible CPU kernel from the detected CPU features, tensor type, operation shape, and active backend priority.
|
| 618 |
+
|
| 619 |
+
Supported targets:
|
| 620 |
+
|
| 621 |
+
| Platform | Supported ABI / architecture | Notes |
|
| 622 |
+
| --- | --- | --- |
|
| 623 |
+
| Linux | AArch64 / arm64 | Runtime CPU feature detection is automatic. |
|
| 624 |
+
| Android | `arm64-v8a` | Use the Android NDK command below for a portable build. |
|
| 625 |
+
| Apple | arm64 | Runtime CPU feature detection is automatic. Non-streaming SVE vector length is treated as unavailable. |
|
| 626 |
+
| Windows | arm64 | Runtime CPU feature detection is automatic. SMCU count is treated as unknown until a detection path is verified. |
|
| 627 |
+
|
| 628 |
+
`GGML_CPU_KLEIDIAI=ON` is valid only for AArch64/arm64 builds. Do not enable it for x86, 32-bit Arm, or Android ABIs other than `arm64-v8a`.
|
| 629 |
+
|
| 630 |
+
### Native AArch64/arm64 build
|
| 631 |
+
|
| 632 |
+
From the llama.cpp source directory:
|
| 633 |
+
|
| 634 |
+
```bash
|
| 635 |
+
cmake -S . -B build -DGGML_CPU_KLEIDIAI=ON
|
| 636 |
+
cmake --build build --config Release
|
| 637 |
+
```
|
| 638 |
+
|
| 639 |
+
### Android arm64-v8a NDK build
|
| 640 |
+
|
| 641 |
+
Set `ANDROID_NDK` to the Android NDK root, then run the following from the llama.cpp source directory. This command configures a portable Android `arm64-v8a` build with KleidiAI enabled and avoids Android dependencies that are not part of the NDK stable native API set.
|
| 642 |
+
|
| 643 |
+
```bash
|
| 644 |
+
cmake -S . -B build-android \
|
| 645 |
+
-DCMAKE_BUILD_TYPE=Release \
|
| 646 |
+
-DCMAKE_TOOLCHAIN_FILE="$ANDROID_NDK/build/cmake/android.toolchain.cmake" \
|
| 647 |
+
-DANDROID_ABI=arm64-v8a \
|
| 648 |
+
-DANDROID_PLATFORM=android-28 \
|
| 649 |
+
-DGGML_CPU_KLEIDIAI=ON \
|
| 650 |
+
-DGGML_NATIVE=OFF \
|
| 651 |
+
-DGGML_OPENMP=OFF \
|
| 652 |
+
-DGGML_LLAMAFILE=OFF \
|
| 653 |
+
-DLLAMA_OPENSSL=OFF
|
| 654 |
+
cmake --build build-android --config Release --parallel
|
| 655 |
+
cmake --install build-android --prefix {install-dir} --config Release
|
| 656 |
+
```
|
| 657 |
+
|
| 658 |
+
Important Android options:
|
| 659 |
+
|
| 660 |
+
- `GGML_CPU_KLEIDIAI=ON` enables KleidiAI for Android `arm64-v8a`.
|
| 661 |
+
- `GGML_NATIVE=OFF` is required for cross-compilation because the build host CPU is not the Android target CPU.
|
| 662 |
+
- `GGML_OPENMP=OFF` avoids adding an OpenMP runtime dependency to this NDK command-line build.
|
| 663 |
+
- `GGML_LLAMAFILE=OFF` avoids the llamafile backend, which is not supported on Android.
|
| 664 |
+
- `LLAMA_OPENSSL=OFF` avoids depending on OpenSSL, which is not part of the Android NDK stable native API set.
|
| 665 |
+
|
| 666 |
+
The Android Studio project under `examples/llama.android` enables KleidiAI automatically for `arm64-v8a`. For Android command-line CMake builds on `arm64-v8a`, pass `-DGGML_CPU_KLEIDIAI=ON` explicitly.
|
| 667 |
+
|
| 668 |
+
Global -march flags such as `-march=armv8.7a` flag are not required for a portable Android `arm64-v8a` build. Global `-march` flags raise the baseline instruction set for generic code. No manual architecture-specific source selection is required; llama.cpp selects compatible KleidiAI kernels at runtime. The KleidiAI libraries internal CMake handles the -march flags for each particular kernel.
|
| 669 |
+
|
| 670 |
+
### Verifying the build
|
| 671 |
+
|
| 672 |
+
Run an installed or in-tree binary:
|
| 673 |
+
|
| 674 |
+
```bash
|
| 675 |
+
./build/bin/llama-cli -m PATH_TO_MODEL -p "What is a car?"
|
| 676 |
+
```
|
| 677 |
+
|
| 678 |
+
If KleidiAI is enabled, the output contains a line similar to:
|
| 679 |
+
|
| 680 |
+
```
|
| 681 |
+
load_tensors: CPU_KLEIDIAI model buffer size = 3474.00 MiB
|
| 682 |
+
```
|
| 683 |
+
|
| 684 |
+
This confirms that the model has tensors allocated through the KleidiAI CPU buffer. It does not prove that every operation, or any specific SME-family operation, used a KleidiAI microkernel. Runtime CPU features, tensor type, operation shape, and backend priority still control dispatch.
|
| 685 |
+
|
| 686 |
+
Depending on the build target, another backend may have higher priority than the CPU backend. To force CPU execution for a run, disable higher priority backends at build time, for example `-DGGML_METAL=OFF`, or use a runtime device option such as `--device none` where supported.
|
| 687 |
+
|
| 688 |
+
### Runtime dispatch
|
| 689 |
+
|
| 690 |
+
KleidiAI microkernels use Arm CPU features such as dotprod, i8mm, SVE, and SME/SME2. Build-time configuration makes the kernels available. Runtime dispatch selects a compatible kernel for the detected CPU and operation. Older or lower-feature CPUs fall back automatically to compatible kernels.
|
| 691 |
+
|
| 692 |
+
KleidiAI accelerates selected `GGML_OP_MUL_MAT` paths for F32 and common quantized formats. Exact coverage depends on the bundled KleidiAI version and the llama.cpp runtime selector, so unsupported tensor types, unsupported operation shapes, or higher priority backends may bypass KleidiAI even when the CPU supports the required Arm feature. This is also why a model may not use SME-family kernels on SME-capable hardware.
|
| 693 |
+
|
| 694 |
+
The current llama.cpp KleidiAI SVE selector only enables SVE kernels when the runtime SVE vector length is known to be QK8_0 bytes, currently 32 bytes. Linux and Android query this at runtime. Apple reports SVE capability separately from userspace non-streaming SVE availability, so llama.cpp treats the SVE vector length as unknown there. Windows exposes SVE feature presence but not the runtime SVE vector length used by this selector, so that value is also treated as unknown. Windows arm64 also treats SMCU count as unknown until a detection mechanism is verified.
|
| 695 |
+
|
| 696 |
+
The set of available SME-family kernels depends on the bundled KleidiAI version and the detected CPU capabilities. Production configuration does not require any KleidiAI runtime environment variables.
|
| 697 |
+
|
| 698 |
+
### Diagnostics and debug overrides
|
| 699 |
+
|
| 700 |
+
KleidiAI runtime environment variables are diagnostics/debug overrides, not production configuration. Leave them unset for normal use.
|
| 701 |
+
|
| 702 |
+
`GGML_KLEIDIAI_SME` controls SME-family kernel selection and overrides the maximum number of threads assigned to selected quantized SME-family kernels:
|
| 703 |
+
|
| 704 |
+
- Not set: use automatic runtime detection.
|
| 705 |
+
- `0`: disable SME-family kernels.
|
| 706 |
+
- `<n> > 0`: enable compatible SME-family kernels and allow up to `<n>` threads for quantized SME-family kernels.
|
| 707 |
+
|
| 708 |
+
On Windows arm64, use `GGML_KLEIDIAI_SME=<n>` as the temporary diagnostics/debug override for SME thread-cap calibration until automatic SMCU count detection is verified.
|
| 709 |
+
|
| 710 |
+
If the CPU does not support the required SME-family capability for a bundled kernel, that kernel is disabled regardless of the environment variable.
|
| 711 |
+
|
| 712 |
+
## OpenCL
|
| 713 |
+
|
| 714 |
+
This provides GPU acceleration through OpenCL on recent Adreno GPU.
|
| 715 |
+
More information about OpenCL backend can be found in [OPENCL.md](./backend/OPENCL.md) for more information.
|
| 716 |
+
|
| 717 |
+
### Android
|
| 718 |
+
|
| 719 |
+
Assume NDK is available in `$ANDROID_NDK`. First, install OpenCL headers and ICD loader library if not available,
|
| 720 |
+
|
| 721 |
+
```sh
|
| 722 |
+
mkdir -p ~/dev/llm
|
| 723 |
+
cd ~/dev/llm
|
| 724 |
+
|
| 725 |
+
git clone https://github.com/KhronosGroup/OpenCL-Headers && \
|
| 726 |
+
cd OpenCL-Headers && \
|
| 727 |
+
cp -r CL $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include
|
| 728 |
+
|
| 729 |
+
cd ~/dev/llm
|
| 730 |
+
|
| 731 |
+
git clone https://github.com/KhronosGroup/OpenCL-ICD-Loader && \
|
| 732 |
+
cd OpenCL-ICD-Loader && \
|
| 733 |
+
mkdir build_ndk && cd build_ndk && \
|
| 734 |
+
cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Release \
|
| 735 |
+
-DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK/build/cmake/android.toolchain.cmake \
|
| 736 |
+
-DOPENCL_ICD_LOADER_HEADERS_DIR=$ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/include \
|
| 737 |
+
-DANDROID_ABI=arm64-v8a \
|
| 738 |
+
-DANDROID_PLATFORM=24 \
|
| 739 |
+
-DANDROID_STL=c++_shared && \
|
| 740 |
+
ninja && \
|
| 741 |
+
cp libOpenCL.so $ANDROID_NDK/toolchains/llvm/prebuilt/linux-x86_64/sysroot/usr/lib/aarch64-linux-android
|
| 742 |
+
```
|
| 743 |
+
|
| 744 |
+
Then build llama.cpp with OpenCL enabled,
|
| 745 |
+
|
| 746 |
+
```sh
|
| 747 |
+
cd ~/dev/llm
|
| 748 |
+
|
| 749 |
+
git clone https://github.com/ggml-org/llama.cpp && \
|
| 750 |
+
cd llama.cpp && \
|
| 751 |
+
mkdir build-android && cd build-android
|
| 752 |
+
|
| 753 |
+
cmake .. -G Ninja \
|
| 754 |
+
-DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK/build/cmake/android.toolchain.cmake \
|
| 755 |
+
-DANDROID_ABI=arm64-v8a \
|
| 756 |
+
-DANDROID_PLATFORM=android-28 \
|
| 757 |
+
-DBUILD_SHARED_LIBS=OFF \
|
| 758 |
+
-DGGML_OPENCL=ON
|
| 759 |
+
|
| 760 |
+
ninja
|
| 761 |
+
```
|
| 762 |
+
|
| 763 |
+
### Windows Arm64
|
| 764 |
+
|
| 765 |
+
First, install OpenCL headers and ICD loader library if not available,
|
| 766 |
+
|
| 767 |
+
```powershell
|
| 768 |
+
mkdir -p ~/dev/llm
|
| 769 |
+
|
| 770 |
+
cd ~/dev/llm
|
| 771 |
+
git clone https://github.com/KhronosGroup/OpenCL-Headers && cd OpenCL-Headers
|
| 772 |
+
mkdir build && cd build
|
| 773 |
+
cmake .. -G Ninja `
|
| 774 |
+
-DBUILD_TESTING=OFF `
|
| 775 |
+
-DOPENCL_HEADERS_BUILD_TESTING=OFF `
|
| 776 |
+
-DOPENCL_HEADERS_BUILD_CXX_TESTS=OFF `
|
| 777 |
+
-DCMAKE_INSTALL_PREFIX="$HOME/dev/llm/opencl"
|
| 778 |
+
cmake --build . --target install
|
| 779 |
+
|
| 780 |
+
cd ~/dev/llm
|
| 781 |
+
git clone https://github.com/KhronosGroup/OpenCL-ICD-Loader && cd OpenCL-ICD-Loader
|
| 782 |
+
mkdir build && cd build
|
| 783 |
+
cmake .. -G Ninja `
|
| 784 |
+
-DCMAKE_BUILD_TYPE=Release `
|
| 785 |
+
-DCMAKE_PREFIX_PATH="$HOME/dev/llm/opencl" `
|
| 786 |
+
-DCMAKE_INSTALL_PREFIX="$HOME/dev/llm/opencl"
|
| 787 |
+
cmake --build . --target install
|
| 788 |
+
```
|
| 789 |
+
|
| 790 |
+
Then build llama.cpp with OpenCL enabled,
|
| 791 |
+
|
| 792 |
+
```powershell
|
| 793 |
+
cmake .. -G Ninja `
|
| 794 |
+
-DCMAKE_TOOLCHAIN_FILE="$HOME/dev/llm/llama.cpp/cmake/arm64-windows-llvm.cmake" `
|
| 795 |
+
-DCMAKE_BUILD_TYPE=Release `
|
| 796 |
+
-DCMAKE_PREFIX_PATH="$HOME/dev/llm/opencl" `
|
| 797 |
+
-DBUILD_SHARED_LIBS=OFF `
|
| 798 |
+
-DGGML_OPENCL=ON
|
| 799 |
+
ninja
|
| 800 |
+
```
|
| 801 |
+
|
| 802 |
+
## Android
|
| 803 |
+
|
| 804 |
+
To read documentation for how to build on Android, [click here](./android.md)
|
| 805 |
+
|
| 806 |
+
## WebGPU
|
| 807 |
+
|
| 808 |
+
The WebGPU backend relies on [Dawn](https://dawn.googlesource.com/dawn). Follow the instructions [here](https://dawn.googlesource.com/dawn/+/refs/heads/main/docs/quickstart-cmake.md) to install Dawn locally so that llama.cpp can find it using CMake. The current implementation is up-to-date with Dawn commit `18eb229`.
|
| 809 |
+
|
| 810 |
+
In the llama.cpp directory, build with CMake:
|
| 811 |
+
|
| 812 |
+
```
|
| 813 |
+
cmake -B build -DGGML_WEBGPU=ON
|
| 814 |
+
cmake --build build --config Release
|
| 815 |
+
```
|
| 816 |
+
|
| 817 |
+
### Browser Support
|
| 818 |
+
|
| 819 |
+
WebGPU allows cross-platform access to the GPU from supported browsers. We utilize [Emscripten](https://emscripten.org/) to compile ggml's WebGPU backend to WebAssembly. Emscripten does not officially support WebGPU bindings yet, but Dawn currently maintains its own WebGPU bindings called emdawnwebgpu.
|
| 820 |
+
|
| 821 |
+
Follow the instructions [here](https://dawn.googlesource.com/dawn/+/refs/heads/main/src/emdawnwebgpu/) to download or build the emdawnwebgpu package (Note that it might be safer to build the emdawnwebgpu package locally, so that it stays in sync with the version of Dawn you have installed above). When building using CMake, the path to the emdawnwebgpu port file needs to be set with the flag `EMDAWNWEBGPU_DIR`.
|
| 822 |
+
|
| 823 |
+
## IBM Z & LinuxONE
|
| 824 |
+
|
| 825 |
+
To read documentation for how to build on IBM Z & LinuxONE, [click here](./build-s390x.md)
|
| 826 |
+
|
| 827 |
+
## OpenVINO
|
| 828 |
+
|
| 829 |
+
[OpenVINO](https://docs.openvino.ai/) is an open-source toolkit for optimizing and deploying high-performance AI inference, specifically designed for Intel hardware (CPUs, GPUs, and NPUs).
|
| 830 |
+
|
| 831 |
+
For build instructions and usage examples, refer to [OPENVINO.md](backend/OPENVINO.md).
|
| 832 |
+
|
| 833 |
+
### Hexagon
|
| 834 |
+
|
| 835 |
+
Check [README.md](./backend/snapdragon/README.md) for target specific build and run info.
|
| 836 |
+
|
| 837 |
+
---
|
| 838 |
+
## Notes about GPU-accelerated backends
|
| 839 |
+
|
| 840 |
+
The GPU may still be used to accelerate some parts of the computation even when using the `-ngl 0` option. You can fully disable GPU acceleration by using `--device none`.
|
| 841 |
+
|
| 842 |
+
In most cases, it is possible to build and use multiple backends at the same time. For example, you can build llama.cpp with both CUDA and Vulkan support by using the `-DGGML_CUDA=ON -DGGML_VULKAN=ON` options with CMake. At runtime, you can specify which backend devices to use with the `--device` option. To see a list of available devices, use the `--list-devices` option.
|
| 843 |
+
|
| 844 |
+
Backends can be built as dynamic libraries that can be loaded dynamically at runtime. This allows you to use the same llama.cpp binary on different machines with different GPUs. To enable this feature, use the `GGML_BACKEND_DL` option when building.
|
examples/llama.android/app/build.gradle.kts
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
plugins {
|
| 2 |
+
alias(libs.plugins.android.application)
|
| 3 |
+
alias(libs.plugins.jetbrains.kotlin.android)
|
| 4 |
+
}
|
| 5 |
+
|
| 6 |
+
android {
|
| 7 |
+
namespace = "com.example.llama"
|
| 8 |
+
compileSdk = 36
|
| 9 |
+
|
| 10 |
+
defaultConfig {
|
| 11 |
+
applicationId = "com.example.llama.aichat"
|
| 12 |
+
|
| 13 |
+
minSdk = 33
|
| 14 |
+
targetSdk = 36
|
| 15 |
+
|
| 16 |
+
versionCode = 1
|
| 17 |
+
versionName = "1.0"
|
| 18 |
+
|
| 19 |
+
testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
|
| 20 |
+
vectorDrawables {
|
| 21 |
+
useSupportLibrary = true
|
| 22 |
+
}
|
| 23 |
+
}
|
| 24 |
+
|
| 25 |
+
buildTypes {
|
| 26 |
+
debug {
|
| 27 |
+
isMinifyEnabled = true
|
| 28 |
+
isShrinkResources = true
|
| 29 |
+
proguardFiles(
|
| 30 |
+
getDefaultProguardFile("proguard-android.txt"),
|
| 31 |
+
"proguard-rules.pro"
|
| 32 |
+
)
|
| 33 |
+
}
|
| 34 |
+
release {
|
| 35 |
+
isMinifyEnabled = true
|
| 36 |
+
isShrinkResources = true
|
| 37 |
+
proguardFiles(
|
| 38 |
+
getDefaultProguardFile("proguard-android-optimize.txt"),
|
| 39 |
+
"proguard-rules.pro"
|
| 40 |
+
)
|
| 41 |
+
}
|
| 42 |
+
}
|
| 43 |
+
compileOptions {
|
| 44 |
+
sourceCompatibility = JavaVersion.VERSION_17
|
| 45 |
+
targetCompatibility = JavaVersion.VERSION_17
|
| 46 |
+
}
|
| 47 |
+
}
|
| 48 |
+
|
| 49 |
+
dependencies {
|
| 50 |
+
implementation(libs.bundles.androidx)
|
| 51 |
+
implementation(libs.material)
|
| 52 |
+
|
| 53 |
+
implementation(project(":lib"))
|
| 54 |
+
|
| 55 |
+
testImplementation(libs.junit)
|
| 56 |
+
androidTestImplementation(libs.androidx.junit)
|
| 57 |
+
androidTestImplementation(libs.androidx.espresso.core)
|
| 58 |
+
}
|
examples/llama.android/build.gradle.kts
ADDED
|
@@ -0,0 +1,6 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
// Top-level build file where you can add configuration options common to all sub-projects/modules.
|
| 2 |
+
plugins {
|
| 3 |
+
alias(libs.plugins.android.application) apply false
|
| 4 |
+
alias(libs.plugins.android.library) apply false
|
| 5 |
+
alias(libs.plugins.jetbrains.kotlin.android) apply false
|
| 6 |
+
}
|
examples/llama.android/lib/build.gradle.kts
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
plugins {
|
| 2 |
+
alias(libs.plugins.android.library)
|
| 3 |
+
alias(libs.plugins.jetbrains.kotlin.android)
|
| 4 |
+
}
|
| 5 |
+
|
| 6 |
+
android {
|
| 7 |
+
namespace = "com.arm.aichat"
|
| 8 |
+
compileSdk = 36
|
| 9 |
+
|
| 10 |
+
ndkVersion = "29.0.13113456"
|
| 11 |
+
|
| 12 |
+
defaultConfig {
|
| 13 |
+
minSdk = 33
|
| 14 |
+
|
| 15 |
+
testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner"
|
| 16 |
+
consumerProguardFiles("consumer-rules.pro")
|
| 17 |
+
|
| 18 |
+
ndk {
|
| 19 |
+
abiFilters += listOf("arm64-v8a", "x86_64")
|
| 20 |
+
}
|
| 21 |
+
externalNativeBuild {
|
| 22 |
+
cmake {
|
| 23 |
+
arguments += "-DCMAKE_BUILD_TYPE=Release"
|
| 24 |
+
arguments += "-DCMAKE_MESSAGE_LOG_LEVEL=DEBUG"
|
| 25 |
+
arguments += "-DCMAKE_VERBOSE_MAKEFILE=ON"
|
| 26 |
+
|
| 27 |
+
arguments += "-DBUILD_SHARED_LIBS=ON"
|
| 28 |
+
arguments += "-DLLAMA_BUILD_APP=OFF"
|
| 29 |
+
arguments += "-DLLAMA_BUILD_COMMON=ON"
|
| 30 |
+
arguments += "-DLLAMA_OPENSSL=OFF"
|
| 31 |
+
|
| 32 |
+
arguments += "-DGGML_NATIVE=OFF"
|
| 33 |
+
arguments += "-DGGML_BACKEND_DL=ON"
|
| 34 |
+
arguments += "-DGGML_CPU_ALL_VARIANTS=ON"
|
| 35 |
+
arguments += "-DGGML_LLAMAFILE=OFF"
|
| 36 |
+
}
|
| 37 |
+
}
|
| 38 |
+
aarMetadata {
|
| 39 |
+
minCompileSdk = 35
|
| 40 |
+
}
|
| 41 |
+
}
|
| 42 |
+
externalNativeBuild {
|
| 43 |
+
cmake {
|
| 44 |
+
path("src/main/cpp/CMakeLists.txt")
|
| 45 |
+
version = "3.31.6"
|
| 46 |
+
}
|
| 47 |
+
}
|
| 48 |
+
compileOptions {
|
| 49 |
+
sourceCompatibility = JavaVersion.VERSION_17
|
| 50 |
+
targetCompatibility = JavaVersion.VERSION_17
|
| 51 |
+
}
|
| 52 |
+
kotlin {
|
| 53 |
+
jvmToolchain(17)
|
| 54 |
+
|
| 55 |
+
compileOptions {
|
| 56 |
+
targetCompatibility = JavaVersion.VERSION_17
|
| 57 |
+
}
|
| 58 |
+
}
|
| 59 |
+
|
| 60 |
+
packaging {
|
| 61 |
+
resources {
|
| 62 |
+
excludes += "/META-INF/{AL2.0,LGPL2.1}"
|
| 63 |
+
}
|
| 64 |
+
}
|
| 65 |
+
|
| 66 |
+
publishing {
|
| 67 |
+
singleVariant("release") {
|
| 68 |
+
withJavadocJar()
|
| 69 |
+
}
|
| 70 |
+
}
|
| 71 |
+
}
|
| 72 |
+
|
| 73 |
+
dependencies {
|
| 74 |
+
implementation(libs.androidx.core.ktx)
|
| 75 |
+
implementation(libs.androidx.datastore.preferences)
|
| 76 |
+
|
| 77 |
+
testImplementation(libs.junit)
|
| 78 |
+
androidTestImplementation(libs.androidx.junit)
|
| 79 |
+
}
|
examples/sycl/build.sh
ADDED
|
@@ -0,0 +1,53 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env bash
|
| 2 |
+
# MIT license
|
| 3 |
+
# Copyright (C) 2024 Intel Corporation
|
| 4 |
+
# SPDX-License-Identifier: MIT
|
| 5 |
+
|
| 6 |
+
print_usage() {
|
| 7 |
+
echo "Usage: ./build.sh [fp32|fp16] [--help]"
|
| 8 |
+
echo ""
|
| 9 |
+
echo "Options:"
|
| 10 |
+
echo " fp32 Build with FP32 precision (default)"
|
| 11 |
+
echo " fp16 Build with FP16 precision (faster for long-prompt inference)"
|
| 12 |
+
echo " --help Print this help message"
|
| 13 |
+
}
|
| 14 |
+
|
| 15 |
+
PRECISION=fp32
|
| 16 |
+
|
| 17 |
+
for arg in "$@"; do
|
| 18 |
+
case "$arg" in
|
| 19 |
+
--help)
|
| 20 |
+
print_usage
|
| 21 |
+
exit 0
|
| 22 |
+
;;
|
| 23 |
+
fp32|fp16)
|
| 24 |
+
PRECISION="$arg"
|
| 25 |
+
;;
|
| 26 |
+
*)
|
| 27 |
+
echo "Error: unknown option '$arg'"
|
| 28 |
+
print_usage
|
| 29 |
+
exit 1
|
| 30 |
+
;;
|
| 31 |
+
esac
|
| 32 |
+
done
|
| 33 |
+
|
| 34 |
+
mkdir -p build
|
| 35 |
+
cd build
|
| 36 |
+
source /opt/intel/oneapi/setvars.sh
|
| 37 |
+
|
| 38 |
+
if [ "$PRECISION" = "fp16" ]; then
|
| 39 |
+
#for FP16
|
| 40 |
+
cmake .. -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DGGML_SYCL_F16=ON -DLLAMA_OPENSSL=OFF # faster for long-prompt inference
|
| 41 |
+
else
|
| 42 |
+
#for FP32
|
| 43 |
+
cmake .. -DGGML_SYCL=ON -DCMAKE_C_COMPILER=icx -DCMAKE_CXX_COMPILER=icpx -DLLAMA_OPENSSL=OFF
|
| 44 |
+
fi
|
| 45 |
+
|
| 46 |
+
#build example/main
|
| 47 |
+
#cmake --build . --config Release --target main
|
| 48 |
+
|
| 49 |
+
#build example/llama-bench
|
| 50 |
+
#cmake --build . --config Release --target llama-bench
|
| 51 |
+
|
| 52 |
+
#build all binary
|
| 53 |
+
cmake --build . --config Release -j$((($(nproc)+1)/2)) -v
|
examples/test-cmake/build-install.sh
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/bash
|
| 2 |
+
|
| 3 |
+
set -e
|
| 4 |
+
|
| 5 |
+
rm -rf llama-build-install install
|
| 6 |
+
|
| 7 |
+
cmake --fresh -S ../../. -B llama-build-install -DCMAKE_BUILD_TYPE=Release \
|
| 8 |
+
-DBUILD_SHARED_LIBS=ON \
|
| 9 |
+
-DGGML_BACKEND_DL=ON \
|
| 10 |
+
-DGGML_CPU_ALL_VARIANTS=ON \
|
| 11 |
+
-DLLAMA_TESTS_INSTALL=OFF \
|
| 12 |
+
-DCMAKE_INSTALL_PREFIX="${PWD}/install" \
|
| 13 |
+
-DGGML_BACKEND_DIR="${PWD}/install/lib/llama.cpp" \
|
| 14 |
+
-DGGML_LIB_INSTALL_DIR="${PWD}/install/lib/llama.cpp" \
|
| 15 |
+
-DLLAMA_LIB_INSTALL_DIR="${PWD}/install/lib/llama.cpp" \
|
| 16 |
+
-DLLAMA_TOOLS_INSTALL=OFF
|
| 17 |
+
|
| 18 |
+
cmake --build llama-build-install --parallel 12
|
| 19 |
+
cmake --install llama-build-install
|
examples/test-cmake/build.sh
ADDED
|
@@ -0,0 +1,7 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/bin/bash
|
| 2 |
+
|
| 3 |
+
set -e
|
| 4 |
+
|
| 5 |
+
cmake -S . -B build -DCMAKE_PREFIX_PATH="${PWD}/install"
|
| 6 |
+
cmake --build build
|
| 7 |
+
LD_LIBRARY_PATH="${PWD}/install/lib/llama.cpp:${PWD}/install/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" ./build/test-cmake
|
models/templates/Apertus-8B-Instruct.jinja
ADDED
|
@@ -0,0 +1,327 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{%- macro render_typescript_type(param_spec, required_params, is_nullable=false) -%}
|
| 2 |
+
{%- if param_spec.type == "array" -%}
|
| 3 |
+
{%- if param_spec['items'] -%}
|
| 4 |
+
{%- if param_spec['items']['type'] == "string" -%}
|
| 5 |
+
{{- "string[]" }}
|
| 6 |
+
{%- elif param_spec['items']['type'] == "number" -%}
|
| 7 |
+
{{- "number[]" }}
|
| 8 |
+
{%- elif param_spec['items']['type'] == "integer" -%}
|
| 9 |
+
{{- "number[]" }}
|
| 10 |
+
{%- elif param_spec['items']['type'] == "boolean" -%}
|
| 11 |
+
{{- "boolean[]" }}
|
| 12 |
+
{%- else -%}
|
| 13 |
+
{%- set inner_type = render_typescript_type(param_spec['items'], required_params) -%}
|
| 14 |
+
{%- if inner_type == "object | object" or inner_type|length > 50 -%}
|
| 15 |
+
{{- "any[]" }}
|
| 16 |
+
{%- else -%}
|
| 17 |
+
{{- inner_type + "[]" }}
|
| 18 |
+
{%- endif -%}
|
| 19 |
+
{%- endif -%}
|
| 20 |
+
{%- if param_spec.nullable -%}
|
| 21 |
+
{{- " | null" }}
|
| 22 |
+
{%- endif -%}
|
| 23 |
+
{%- else -%}
|
| 24 |
+
{{- "any[]" }}
|
| 25 |
+
{%- if param_spec.nullable -%}
|
| 26 |
+
{{- " | null" }}
|
| 27 |
+
{%- endif -%}
|
| 28 |
+
{%- endif -%}
|
| 29 |
+
{%- elif param_spec.type is defined and param_spec.type is iterable and param_spec.type is not string and param_spec.type is not mapping and param_spec.type[0] is defined -%}
|
| 30 |
+
{#- Handle array of types like ["object", "object"] from Union[dict, list] #}
|
| 31 |
+
{%- if param_spec.type | length > 1 -%}
|
| 32 |
+
{{- param_spec.type | join(" | ") }}
|
| 33 |
+
{%- else -%}
|
| 34 |
+
{{- param_spec.type[0] }}
|
| 35 |
+
{%- endif -%}
|
| 36 |
+
{%- elif param_spec.oneOf -%}
|
| 37 |
+
{#- Handle oneOf schemas - check for complex unions and fallback to any #}
|
| 38 |
+
{%- set has_object_variants = false -%}
|
| 39 |
+
{%- for variant in param_spec.oneOf -%}
|
| 40 |
+
{%- if variant.type == "object" -%}
|
| 41 |
+
{%- set has_object_variants = true -%}
|
| 42 |
+
{%- endif -%}
|
| 43 |
+
{%- endfor -%}
|
| 44 |
+
{%- if has_object_variants and param_spec.oneOf|length > 1 -%}
|
| 45 |
+
{{- "any" }}
|
| 46 |
+
{%- else -%}
|
| 47 |
+
{%- for variant in param_spec.oneOf -%}
|
| 48 |
+
{{- render_typescript_type(variant, required_params) -}}
|
| 49 |
+
{%- if variant.description %}
|
| 50 |
+
{{- "// " + variant.description }}
|
| 51 |
+
{%- endif -%}
|
| 52 |
+
{%- if variant.default is defined %}
|
| 53 |
+
{{ "// default: " + variant.default|tojson }}
|
| 54 |
+
{%- endif -%}
|
| 55 |
+
{%- if not loop.last %}
|
| 56 |
+
{{- " | " }}
|
| 57 |
+
{% endif -%}
|
| 58 |
+
{%- endfor -%}
|
| 59 |
+
{%- endif -%}
|
| 60 |
+
{%- elif param_spec.type == "string" -%}
|
| 61 |
+
{%- if param_spec.enum -%}
|
| 62 |
+
{{- '"' + param_spec.enum|join('" | "') + '"' -}}
|
| 63 |
+
{%- else -%}
|
| 64 |
+
{{- "string" }}
|
| 65 |
+
{%- if param_spec.nullable %}
|
| 66 |
+
{{- " | null" }}
|
| 67 |
+
{%- endif -%}
|
| 68 |
+
{%- endif -%}
|
| 69 |
+
{%- elif param_spec.type == "number" -%}
|
| 70 |
+
{{- "number" }}
|
| 71 |
+
{%- elif param_spec.type == "integer" -%}
|
| 72 |
+
{{- "number" }}
|
| 73 |
+
{%- elif param_spec.type == "boolean" -%}
|
| 74 |
+
{{- "boolean" }}
|
| 75 |
+
{%- elif param_spec.type == "object" -%}
|
| 76 |
+
{%- if param_spec.properties -%}
|
| 77 |
+
{{- "{\n" }}
|
| 78 |
+
{%- for prop_name, prop_spec in param_spec.properties.items() -%}
|
| 79 |
+
{{- prop_name -}}
|
| 80 |
+
{%- if prop_name not in (param_spec.required or []) -%}
|
| 81 |
+
{{- "?" }}
|
| 82 |
+
{%- endif -%}
|
| 83 |
+
{{- ": " }}
|
| 84 |
+
{{ render_typescript_type(prop_spec, param_spec.required or []) }}
|
| 85 |
+
{%- if not loop.last -%}
|
| 86 |
+
{{-", " }}
|
| 87 |
+
{%- endif -%}
|
| 88 |
+
{%- endfor -%}
|
| 89 |
+
{{- "}" }}
|
| 90 |
+
{%- else -%}
|
| 91 |
+
{{- "object" }}
|
| 92 |
+
{%- endif -%}
|
| 93 |
+
{%- else -%}
|
| 94 |
+
{{- "any" }}
|
| 95 |
+
{%- endif -%}
|
| 96 |
+
{%- endmacro -%}
|
| 97 |
+
|
| 98 |
+
{%- macro render_tools(tools) -%}
|
| 99 |
+
{%- for tool in tools %}
|
| 100 |
+
{{- "// " + tool.function.description + "\n" }}
|
| 101 |
+
{{- "type "+ tool.function.name + " = " }}
|
| 102 |
+
{%- if tool.function.parameters and tool.function.parameters.properties %}
|
| 103 |
+
{{- "(_: {\n" }}
|
| 104 |
+
{%- for param_name, param_spec in tool.function.parameters.properties.items() %}
|
| 105 |
+
{%- if param_spec.description %}
|
| 106 |
+
{{- "// " + param_spec.description + "\n" }}
|
| 107 |
+
{%- endif %}
|
| 108 |
+
{{- param_name }}
|
| 109 |
+
{%- if param_name not in (tool.function.parameters.required or []) -%}
|
| 110 |
+
{{- "?" }}
|
| 111 |
+
{%- endif -%}
|
| 112 |
+
{{- ": " }}
|
| 113 |
+
{{- render_typescript_type(param_spec, tool.function.parameters.required or []) }}
|
| 114 |
+
{%- if param_spec.default is defined -%}
|
| 115 |
+
{%- if param_spec.enum %}
|
| 116 |
+
{{- ", // default: " + param_spec.default }}
|
| 117 |
+
{%- elif param_spec.oneOf %}
|
| 118 |
+
{{- "// default: " + param_spec.default }}
|
| 119 |
+
{%- else %}
|
| 120 |
+
{{- ", // default: " + param_spec.default|tojson }}
|
| 121 |
+
{%- endif -%}
|
| 122 |
+
{%- endif -%}
|
| 123 |
+
{%- if not loop.last %}
|
| 124 |
+
{{- ",\n" }}
|
| 125 |
+
{%- else %}
|
| 126 |
+
{{- "\n" }}
|
| 127 |
+
{%- endif -%}
|
| 128 |
+
{%- endfor %}
|
| 129 |
+
{{- "}) => any;" }}
|
| 130 |
+
{%- else -%}
|
| 131 |
+
{{- "() => any;" }}
|
| 132 |
+
{%- endif -%}
|
| 133 |
+
{%- if not loop.last -%}
|
| 134 |
+
{{- "\n" }}
|
| 135 |
+
{%- endif -%}
|
| 136 |
+
{%- endfor %}
|
| 137 |
+
{%- endmacro -%}
|
| 138 |
+
|
| 139 |
+
{{ bos_token }}
|
| 140 |
+
|
| 141 |
+
{%- set system_token = '<|system_start|>' -%}
|
| 142 |
+
{%- set end_system_token = '<|system_end|>' -%}
|
| 143 |
+
{%- set developer_token = '<|developer_start|>' -%}
|
| 144 |
+
{%- set end_developer_token = '<|developer_end|>' -%}
|
| 145 |
+
{%- set user_token = '<|user_start|>' -%}
|
| 146 |
+
{%- set end_user_token = '<|user_end|>' -%}
|
| 147 |
+
{%- set assistant_token = '<|assistant_start|>' -%}
|
| 148 |
+
{%- set end_assistant_token = '<|assistant_end|>' -%}
|
| 149 |
+
{%- set inner_token = '<|inner_prefix|>' -%}
|
| 150 |
+
{%- set outer_token = '<|inner_suffix|>' -%}
|
| 151 |
+
{%- set tool_calls_token = '<|tools_prefix|>' -%}
|
| 152 |
+
{%- set end_tool_calls_token = '<|tools_suffix|>' -%}
|
| 153 |
+
|
| 154 |
+
{%- set ns = namespace(in_assistant=false, in_tool=false, in_inner=false, assistant_format=none) -%}
|
| 155 |
+
|
| 156 |
+
{%- if messages and messages[0].role == 'system' -%}
|
| 157 |
+
{%- if "content" in messages[0] -%}
|
| 158 |
+
{%- if messages[0].content is string -%}
|
| 159 |
+
{{ system_token + messages[0].content + end_system_token }}
|
| 160 |
+
{%- elif messages[0].content is mapping and "text" in messages[0].content -%}
|
| 161 |
+
{{ system_token + messages[0].content.text + end_system_token }}
|
| 162 |
+
{%- else -%}
|
| 163 |
+
{{- raise_exception("Invalid system message") -}}
|
| 164 |
+
{%- endif -%}
|
| 165 |
+
{%- else -%}
|
| 166 |
+
{{- raise_exception("Invalid system message") -}}
|
| 167 |
+
{%- endif -%}
|
| 168 |
+
{%- set loop_messages = messages[1:] -%}
|
| 169 |
+
{%- else -%}
|
| 170 |
+
{{ system_token + 'You are Apertus, a helpful assistant created by the SwissAI initiative.\nKnowledge cutoff: 2024-04\nCurrent date: ' + strftime_now('%Y-%m-%d') + end_system_token }}
|
| 171 |
+
{%- set loop_messages = messages -%}
|
| 172 |
+
{%- endif -%}
|
| 173 |
+
|
| 174 |
+
{{ developer_token + 'Deliberation: ' }}
|
| 175 |
+
{%- if enable_thinking is defined and enable_thinking -%}
|
| 176 |
+
{{ 'enabled\n' }}
|
| 177 |
+
{%- else -%}
|
| 178 |
+
{{ 'disabled\n' }}
|
| 179 |
+
{%- endif -%}
|
| 180 |
+
{%- if tools is defined and tools -%}
|
| 181 |
+
{{ 'Tool Capabilities:\n' + render_tools(tools) }}
|
| 182 |
+
{%- else -%}
|
| 183 |
+
{{ 'Tool Capabilities: disabled' }}
|
| 184 |
+
{%- endif -%}
|
| 185 |
+
{{ end_developer_token }}
|
| 186 |
+
|
| 187 |
+
{%- for message in loop_messages -%}
|
| 188 |
+
{%- if message.role == 'user' -%}
|
| 189 |
+
{%- set ns.in_inner = false -%}
|
| 190 |
+
{%- if ns.in_tool -%}
|
| 191 |
+
{{ ']' }}
|
| 192 |
+
{%- set ns.in_tool = false -%}
|
| 193 |
+
{%- endif -%}
|
| 194 |
+
{%- if ns.in_assistant -%}
|
| 195 |
+
{{ end_assistant_token }}
|
| 196 |
+
{%- set ns.in_assistant = false -%}
|
| 197 |
+
{%- endif -%}
|
| 198 |
+
{%- if "content" in message -%}
|
| 199 |
+
{{ user_token }}
|
| 200 |
+
{%- if message.content is string -%}
|
| 201 |
+
{{ message.content }}
|
| 202 |
+
{%- elif message.content is mapping and "parts" in message.content -%}
|
| 203 |
+
{%- set parts = message.content.parts -%}
|
| 204 |
+
{%- for part in parts -%}
|
| 205 |
+
{%- if part.type == "text" -%}
|
| 206 |
+
{{ part.text }}
|
| 207 |
+
{%- else -%}
|
| 208 |
+
{{- raise_exception("Invalid user part: " + part.type) -}}
|
| 209 |
+
{%- endif -%}
|
| 210 |
+
{%- endfor -%}
|
| 211 |
+
{%- else -%}
|
| 212 |
+
{{- raise_exception("Invalid user message: " + message.role) -}}
|
| 213 |
+
{%- endif -%}
|
| 214 |
+
{{ end_user_token }}
|
| 215 |
+
{%- endif -%}
|
| 216 |
+
{%- elif message.role == 'assistant' -%}
|
| 217 |
+
{%- if not ns.in_assistant -%}
|
| 218 |
+
{{ assistant_token }}
|
| 219 |
+
{%- set ns.in_assistant = true -%}
|
| 220 |
+
{%- endif -%}
|
| 221 |
+
{%- if "content" in message and message.content is not none -%}
|
| 222 |
+
{%- if message.content is string and (ns.assistant_format is none or ns.assistant_format == "string") -%}
|
| 223 |
+
{%- if ns.in_tool -%}
|
| 224 |
+
{{ ']' }}
|
| 225 |
+
{%- set ns.in_tool = false -%}
|
| 226 |
+
{%- endif -%}
|
| 227 |
+
{%- set ns.assistant_format = "string" -%}
|
| 228 |
+
{{ message.content }}
|
| 229 |
+
{%- elif message.content is mapping and "blocks" in message.content and (ns.assistant_format is none or ns.assistant_format == "mapping") -%}
|
| 230 |
+
{%- set ns.assistant_format = "mapping" -%}
|
| 231 |
+
{%- set blocks = message.content.blocks -%}
|
| 232 |
+
{%- for block in blocks -%}
|
| 233 |
+
{%- if block.type == 'thoughts' -%}
|
| 234 |
+
{%- if ns.in_tool -%}
|
| 235 |
+
{{ ']' }}
|
| 236 |
+
{%- set ns.in_tool = false -%}
|
| 237 |
+
{%- endif -%}
|
| 238 |
+
{%- if not ns.in_inner -%}
|
| 239 |
+
{%- set ns.in_inner = true -%}
|
| 240 |
+
{{ inner_token }}
|
| 241 |
+
{%- endif -%}
|
| 242 |
+
{{ block.text }}
|
| 243 |
+
{%- elif block.type == 'tool_calls' -%}
|
| 244 |
+
{%- if ns.in_tool -%}
|
| 245 |
+
{{ ']' }}
|
| 246 |
+
{%- set ns.in_tool = false -%}
|
| 247 |
+
{%- endif -%}
|
| 248 |
+
{%- if ns.in_inner and not loop.first and block.calls|length == 1 and block.calls[0].name == 'display_answers' -%}
|
| 249 |
+
{%- set ns.in_inner = false -%}
|
| 250 |
+
{{ outer_token }}
|
| 251 |
+
{%- endif -%}
|
| 252 |
+
{{ tool_calls_token + '[' }}
|
| 253 |
+
{%- for tool_call in block.calls -%}
|
| 254 |
+
{{- '{"' + tool_call.name + '": ' + tool_call.arguments + '}' }}
|
| 255 |
+
{%- if not loop.last -%}
|
| 256 |
+
{{- ", " }}
|
| 257 |
+
{%- endif -%}
|
| 258 |
+
{%- endfor -%}
|
| 259 |
+
{{ ']' + end_tool_calls_token }}
|
| 260 |
+
{%- elif block.type == 'tool_outputs' -%}
|
| 261 |
+
{%- if ns.in_tool -%}
|
| 262 |
+
{{- raise_exception("Cannot have both tool outputs as separate messages and tool outputs as blocks") -}}
|
| 263 |
+
{%- endif -%}
|
| 264 |
+
{{ '[' }}
|
| 265 |
+
{%- for tool_output in block.outputs -%}
|
| 266 |
+
{{- tool_output.output }}
|
| 267 |
+
{%- if not loop.last -%}
|
| 268 |
+
{{- ", " }}
|
| 269 |
+
{%- endif -%}
|
| 270 |
+
{%- endfor -%}
|
| 271 |
+
{{- ']' }}
|
| 272 |
+
{%- elif block.type == 'response' -%}
|
| 273 |
+
{%- if ns.in_tool -%}
|
| 274 |
+
{{ ']' }}
|
| 275 |
+
{%- set ns.in_tool = false -%}
|
| 276 |
+
{%- endif -%}
|
| 277 |
+
{%- if (not loop.first and ns.in_inner) or (ns.in_assistant and ns.in_inner) -%}
|
| 278 |
+
{%- set ns.in_inner = false -%}
|
| 279 |
+
{{ outer_token }}
|
| 280 |
+
{%- endif -%}
|
| 281 |
+
{{ block.text }}
|
| 282 |
+
{%- else -%}
|
| 283 |
+
{{- raise_exception("Invalid assistant block type: " + block.type) -}}
|
| 284 |
+
{%- endif -%}
|
| 285 |
+
{%- endfor -%}
|
| 286 |
+
{%- else -%}
|
| 287 |
+
{{- raise_exception("Invalid assistant content '" + message.content + "', expected " + ns.assistant_format) -}}
|
| 288 |
+
{%- endif -%}
|
| 289 |
+
{%- elif "tool_calls" not in message -%}
|
| 290 |
+
{{- raise_exception("Invalid assistant message " + message) -}}
|
| 291 |
+
{%- endif -%}
|
| 292 |
+
{%- if "tool_calls" in message and message.tool_calls -%}
|
| 293 |
+
{{ tool_calls_token + '[' }}
|
| 294 |
+
{%- for tool_call in message.tool_calls -%}
|
| 295 |
+
{%- if tool_call.type == 'function' -%}
|
| 296 |
+
{%- set function = tool_call.function -%}
|
| 297 |
+
{{- '{"' + function.name + '": ' + function.arguments|tojson + '}' }}
|
| 298 |
+
{%- if not loop.last -%}
|
| 299 |
+
{{- ", " }}
|
| 300 |
+
{%- endif -%}
|
| 301 |
+
{%- else -%}
|
| 302 |
+
{{- raise_exception("Invalid tool call type: " + tool_call.type) -}}
|
| 303 |
+
{%- endif -%}
|
| 304 |
+
{%- endfor -%}
|
| 305 |
+
{{ ']' + end_tool_calls_token }}
|
| 306 |
+
{%- endif -%}
|
| 307 |
+
{%- elif message.role == 'tool' -%}
|
| 308 |
+
{%- if not ns.in_assistant -%}
|
| 309 |
+
{{- raise_exception("Tool message outside of assistant") -}}
|
| 310 |
+
{%- endif -%}
|
| 311 |
+
{%- if not ns.in_tool -%}
|
| 312 |
+
{{ '[' }}
|
| 313 |
+
{%- set ns.in_tool = true -%}
|
| 314 |
+
{%- else -%}
|
| 315 |
+
{{ ", "}}
|
| 316 |
+
{%- endif -%}
|
| 317 |
+
{{ message.content }}
|
| 318 |
+
{%- else -%}
|
| 319 |
+
{{- raise_exception("Invalid message role") -}}
|
| 320 |
+
{%- endif -%}
|
| 321 |
+
{%- endfor -%}
|
| 322 |
+
{%- if ns.in_tool -%}
|
| 323 |
+
{{ ']' }}
|
| 324 |
+
{%- endif -%}
|
| 325 |
+
{%- if add_generation_prompt -%}
|
| 326 |
+
{{ assistant_token }}
|
| 327 |
+
{%- endif -%}
|
models/templates/Apriel-1.6-15b-Thinker-fixed.jinja
ADDED
|
@@ -0,0 +1,171 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{# ---------------------------------------------------------------------- #}
|
| 2 |
+
{# ƛƬ Default setup and flags #}
|
| 3 |
+
{# ---------------------------------------------------------------------- #}
|
| 4 |
+
{%- set messages = messages or [] -%}
|
| 5 |
+
{%- set tools = tools or [] -%}
|
| 6 |
+
{%- set add_generation_prompt = add_generation_prompt or false -%}
|
| 7 |
+
{%- set available_tool_string = '' -%}
|
| 8 |
+
{%- set add_tool_id = true -%}
|
| 9 |
+
{%- set add_thoughts = true -%} {# whether to include <thinking> reasoning blocks #}
|
| 10 |
+
{# Optional token placeholders (safe defaults) #}
|
| 11 |
+
{%- set bos_token = bos_token or '' -%}
|
| 12 |
+
{%- set eos_token = eos_token or '' -%}
|
| 13 |
+
{# ---------------------------------------------------------------------- #}
|
| 14 |
+
{# Core reasoning prompt and assistant reasoning prefix #}
|
| 15 |
+
{# ---------------------------------------------------------------------- #}
|
| 16 |
+
{%- set reasoning_prompt -%}
|
| 17 |
+
You are a thoughtful, systematic AI assistant from ServiceNow Language Models (SLAM) lab.
|
| 18 |
+
Analyze each question carefully, present your reasoning step-by-step, then provide the final
|
| 19 |
+
response after the marker [BEGIN FINAL RESPONSE].
|
| 20 |
+
{%- endset -%}
|
| 21 |
+
{%- set reasoning_asst_turn_start = 'Here are my reasoning steps:\n' -%}
|
| 22 |
+
{# ---------------------------------------------------------------------- #}
|
| 23 |
+
{# Tool list and tool call output format #}
|
| 24 |
+
{# ---------------------------------------------------------------------- #}
|
| 25 |
+
{%- if tools|length > 0 -%}
|
| 26 |
+
{%- set available_tool_string -%}
|
| 27 |
+
You are provided with function signatures within <available_tools></available_tools> XML tags.
|
| 28 |
+
You may call one or more functions to assist with the user query.
|
| 29 |
+
Don't make assumptions about the arguments. You should infer the argument values from previous
|
| 30 |
+
user responses and the system message.
|
| 31 |
+
Here are the available tools:
|
| 32 |
+
<available_tools>
|
| 33 |
+
{% for tool in tools %}{{ tool|string }}{% endfor %}
|
| 34 |
+
|
| 35 |
+
</available_tools>.
|
| 36 |
+
|
| 37 |
+
Return all function calls as a list of JSON objects within <tool_calls></tool_calls> XML tags.
|
| 38 |
+
Each JSON object should contain a function name and arguments as follows:
|
| 39 |
+
<tool_calls>[
|
| 40 |
+
{"name": <function-name-1>, "arguments": <args-dict-1>},
|
| 41 |
+
{"name": <function-name-2>, "arguments": <args-dict-2>},
|
| 42 |
+
...
|
| 43 |
+
]</tool_calls>
|
| 44 |
+
{%- endset -%}
|
| 45 |
+
{%- endif -%}
|
| 46 |
+
{# ---------------------------------------------------------------------- #}
|
| 47 |
+
{# Start system block if first message is not system #}
|
| 48 |
+
{# ---------------------------------------------------------------------- #}
|
| 49 |
+
{%- if messages|length > 0 and messages[0]['role'] != 'system' -%}
|
| 50 |
+
{%- if tools|length > 0 -%}
|
| 51 |
+
{{ bos_token + '<|begin_system|>\n' + reasoning_prompt + '\n' + available_tool_string + '\n' }}
|
| 52 |
+
{%- else -%}
|
| 53 |
+
{{ bos_token + '<|begin_system|>\n' + reasoning_prompt + '\n' }}
|
| 54 |
+
{%- endif -%}
|
| 55 |
+
{%- endif -%}
|
| 56 |
+
{# ---------------------------------------------------------------------- #}
|
| 57 |
+
{# Iterate through messages #}
|
| 58 |
+
{# ---------------------------------------------------------------------- #}
|
| 59 |
+
{%- for message in messages -%}
|
| 60 |
+
|
| 61 |
+
{# ---------------- USER MESSAGE ---------------- #}
|
| 62 |
+
{%- if message['role'] == 'user' -%}
|
| 63 |
+
{{ '<|begin_user|>\n' }}
|
| 64 |
+
{%- if message['content'] is not string -%}
|
| 65 |
+
{%- for chunk in message['content'] -%}
|
| 66 |
+
{%- if chunk['type'] == 'text' -%}
|
| 67 |
+
{{ chunk['text'] }}
|
| 68 |
+
{%- elif chunk['type'] in ['image', 'image_url'] -%}
|
| 69 |
+
{{ '[IMG]' }}
|
| 70 |
+
{%- else -%}
|
| 71 |
+
{{ raise_exception('Unrecognized content type!') }}
|
| 72 |
+
{%- endif -%}
|
| 73 |
+
{%- endfor -%}
|
| 74 |
+
{%- else -%}
|
| 75 |
+
{{ message['content'] }}
|
| 76 |
+
{%- endif -%}
|
| 77 |
+
|
| 78 |
+
{# ---------------- SYSTEM MESSAGE ---------------- #}
|
| 79 |
+
{%- elif message['role'] == 'system' -%}
|
| 80 |
+
{%- set sys_content = message.get('content', '') -%}
|
| 81 |
+
{%- if sys_content and sys_content|length > 0 -%}
|
| 82 |
+
{%- if sys_content is string -%}
|
| 83 |
+
{%- set system_message = sys_content -%}
|
| 84 |
+
{%- else -%}
|
| 85 |
+
{%- set system_message = sys_content[0]['text'] -%}
|
| 86 |
+
{%- endif -%}
|
| 87 |
+
{%- else -%}
|
| 88 |
+
{%- set system_message = '' -%}
|
| 89 |
+
{%- endif -%}
|
| 90 |
+
|
| 91 |
+
{%- if tools|length > 0 -%}
|
| 92 |
+
{{ bos_token + '<|begin_system|>\n' + reasoning_prompt + '\n' + system_message + '\n' + available_tool_string + '\n' }}
|
| 93 |
+
{%- else -%}
|
| 94 |
+
{{ bos_token + '<|begin_system|>\n' + reasoning_prompt + '\n' + system_message + '\n' }}
|
| 95 |
+
{%- endif -%}
|
| 96 |
+
|
| 97 |
+
{# ---------------- ASSISTANT MESSAGE ---------------- #}
|
| 98 |
+
{%- elif message['role'] == 'assistant' -%}
|
| 99 |
+
{%- if loop.last -%}
|
| 100 |
+
{%- set add_tool_id = false -%}
|
| 101 |
+
{%- endif -%}
|
| 102 |
+
|
| 103 |
+
{{ '\n<|begin_assistant|>\n' }}
|
| 104 |
+
|
| 105 |
+
{%- if add_thoughts and message.get('reasoning_content') and loop.last -%}
|
| 106 |
+
{{ message['reasoning_content'] + '\n[BEGIN FINAL RESPONSE]\n' }}
|
| 107 |
+
{%- endif -%}
|
| 108 |
+
|
| 109 |
+
{%- set asst_content = message.get('content', '') -%}
|
| 110 |
+
{%- if asst_content and asst_content|length > 0 -%}
|
| 111 |
+
{%- if asst_content is not string -%}
|
| 112 |
+
{%- set asst_text = asst_content[0]['text'] -%}
|
| 113 |
+
{%- else -%}
|
| 114 |
+
{%- set asst_text = asst_content -%}
|
| 115 |
+
{%- endif -%}
|
| 116 |
+
{# For historical turns (not the last), strip reasoning and keep only final response #}
|
| 117 |
+
{%- if not loop.last and '[BEGIN FINAL RESPONSE]' in asst_text -%}
|
| 118 |
+
{{- asst_text.split('[BEGIN FINAL RESPONSE]')[-1] | trim -}}
|
| 119 |
+
{%- else -%}
|
| 120 |
+
{{- asst_text -}}
|
| 121 |
+
{%- endif -%}
|
| 122 |
+
{%- elif message.get('chosen') and message['chosen']|length > 0 -%}
|
| 123 |
+
{{ message['chosen'][0] }}
|
| 124 |
+
{%- endif -%}
|
| 125 |
+
|
| 126 |
+
{# Tool call output #}
|
| 127 |
+
{%- set tool_calls = message.get('tool_calls', []) -%}
|
| 128 |
+
{%- if tool_calls and tool_calls|length > 0 -%}
|
| 129 |
+
{{ '\n<tool_calls>[' }}
|
| 130 |
+
{%- for tool_call in tool_calls -%}
|
| 131 |
+
{{ '{"name": "' + tool_call['function']['name'] + '", "arguments": ' + tool_call['function']['arguments']|tojson }}
|
| 132 |
+
{%- if add_tool_id == true and 'id' in tool_call -%}
|
| 133 |
+
{{ ', "id": "' + tool_call['id'] + '"' }}
|
| 134 |
+
{%- endif -%}
|
| 135 |
+
{{ '}' }}
|
| 136 |
+
{%- if not loop.last -%}{{ ', ' }}{%- endif -%}
|
| 137 |
+
{%- endfor -%}
|
| 138 |
+
{{ ']</tool_calls>' }}
|
| 139 |
+
{%- endif -%}
|
| 140 |
+
|
| 141 |
+
{%- set training_prompt = training_prompt if (training_prompt is defined) else false -%}
|
| 142 |
+
{%- if not loop.last or training_prompt -%}
|
| 143 |
+
{{ '\n<|end|>\n' }}
|
| 144 |
+
{%- endif -%}
|
| 145 |
+
|
| 146 |
+
{# ---------------- TOOL RESULT MESSAGE ---------------- #}
|
| 147 |
+
{%- elif message['role'] == 'tool' -%}
|
| 148 |
+
{%- set tool_content = message.get('content', '') -%}
|
| 149 |
+
{%- if tool_content is string -%}
|
| 150 |
+
{%- set tool_message = tool_content -%}
|
| 151 |
+
{%- else -%}
|
| 152 |
+
{%- set tool_message = tool_content[0]['text'] if tool_content else '' -%}
|
| 153 |
+
{%- endif -%}
|
| 154 |
+
{{ '<|begin_tool_result|>\n' + tool_message|string + '\n' }}
|
| 155 |
+
|
| 156 |
+
{# ---------------- CONTENT MESSAGE ---------------- #}
|
| 157 |
+
{%- elif message['role'] == 'content' -%}
|
| 158 |
+
{%- set msg_content = message.get('content', '') -%}
|
| 159 |
+
{%- if msg_content is not string -%}
|
| 160 |
+
{{ '<|begin_content|>\n' + msg_content[0]['text'] + '\n' }}
|
| 161 |
+
{%- else -%}
|
| 162 |
+
{{ '<|begin_content|>\n' + msg_content + '\n' }}
|
| 163 |
+
{%- endif -%}
|
| 164 |
+
{%- endif -%}
|
| 165 |
+
|
| 166 |
+
{# ---------------- REASONING PROMPT BEFORE NEXT ASSISTANT ---------------- #}
|
| 167 |
+
{%- if loop.last and add_generation_prompt and message['role'] != 'assistant' -%}
|
| 168 |
+
{{ '\n<|begin_assistant|>\n' + reasoning_asst_turn_start }}
|
| 169 |
+
{%- endif -%}
|
| 170 |
+
|
| 171 |
+
{%- endfor -%}
|
models/templates/Bielik-11B-v3.0-Instruct.jinja
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{{ bos_token }}
|
| 2 |
+
{%- if messages[0]['role'] == 'system' %}
|
| 3 |
+
{%- set system_message = messages[0]['content'] %}
|
| 4 |
+
{%- set loop_start_index = 1 %}
|
| 5 |
+
{%- else %}
|
| 6 |
+
{%- set system_message = "" %}
|
| 7 |
+
{%- set loop_start_index = 0 %}
|
| 8 |
+
{%- endif %}
|
| 9 |
+
|
| 10 |
+
{%- if system_message or tools %}
|
| 11 |
+
{{- '<|im_start|>system\n' }}
|
| 12 |
+
{%- if system_message %}
|
| 13 |
+
{{- system_message }}
|
| 14 |
+
{%- endif %}
|
| 15 |
+
{%- if tools %}
|
| 16 |
+
{{- '\n\nMasz dostęp do następujących narzędzi. Definicje narzędzi znajdują się poniżej wewnątrz znaczników <|function_list|>:\n<|function_list|>\n' }}
|
| 17 |
+
{{- '[' }}
|
| 18 |
+
{%- for tool in tools %}
|
| 19 |
+
{{- tool | tojson }}
|
| 20 |
+
{%- if not loop.last %}
|
| 21 |
+
{{- ',\n' }}
|
| 22 |
+
{%- endif %}
|
| 23 |
+
{%- endfor %}
|
| 24 |
+
{{- ']\n<|function_list|>\n\nAby wywołać narzędzie, użyj formatu <tool_call>{"name": "nazwa_narzędzia", "arguments": {"argument": "wartość"}}</tool_call>. Wyniki działania narzędzi zostaną przekazane z markerem <|function_output|>.\n' }}
|
| 25 |
+
{%- endif %}
|
| 26 |
+
{%- if enable_thinking %}
|
| 27 |
+
{{- '\n\nZanim odpowiesz na pytanie, najpierw przemyśl swoje kroki i umieść swoje myśli wewnątrz tagów <think>...</think>. Musisz najpierw pomyśleć, zanim udzielisz odpowiedzi. ' -}}
|
| 28 |
+
{{- 'WAŻNE: Powinieneś myśleć w tym samym języku, co pytanie użytkownika. Jeśli pytanie jest zadane po polsku, powinieneś również myśleć po polsku. Jeśli pytanie jest po angielsku, myślisz również po angielsku itd. ' }}
|
| 29 |
+
{{- '** PAMIĘTAJ! ** Pytanie po polsku -> myślenie po polsku -> odpowiedź po polsku!' -}}
|
| 30 |
+
{{- '*** BARDZO WAŻNE!!! *** Jesteś Bielikiem, polskim modelem językowym. Twoją główną cechą jest umiejętność pisania po polsku. Jeśli użytkownik zadaje Ci pytania po polsku, ZAWSZE odpowiadaj po polsku. ' -}}
|
| 31 |
+
{{- 'Nawet, jeśli korzystasz z narzędzia, którego większość instrukcji jest po angielsku, powinieneś przede wszystkim odpowiadać po polsku, jeśli użytkownik zadaje pytanie w tym języku. ' -}}
|
| 32 |
+
{%- endif %}
|
| 33 |
+
{{- '<|im_end|>\n' }}
|
| 34 |
+
{%- endif %}
|
| 35 |
+
|
| 36 |
+
{%- for message in messages[loop_start_index:] %}
|
| 37 |
+
{%- if message['role'] == 'user' %}
|
| 38 |
+
{{- '<|im_start|>user\n' + message['content'] + '<|im_end|>\n' }}
|
| 39 |
+
{%- elif message['role'] == 'assistant' %}
|
| 40 |
+
{{- '<|im_start|>assistant\n' }}
|
| 41 |
+
{%- set content = message.content | default('') %}
|
| 42 |
+
{%- set reasoning_content = message.reasoning_content | default('') %}
|
| 43 |
+
{%- if not reasoning_content and '<think>' in content and '</think>' in content %}
|
| 44 |
+
{%- set reasoning_parts = content.split('</think>') %}
|
| 45 |
+
{%- set reasoning_content = reasoning_parts[0].split('<think>')[-1] %}
|
| 46 |
+
{%- set content = reasoning_parts[1:] | join('</think>') %}
|
| 47 |
+
{%- endif %}
|
| 48 |
+
{%- if reasoning_content %}
|
| 49 |
+
{{- '<think>\n' + reasoning_content.strip() + '\n</think>\n' }}
|
| 50 |
+
{%- endif %}
|
| 51 |
+
{{- content.lstrip() }}
|
| 52 |
+
{%- if message.tool_calls %}
|
| 53 |
+
{%- for tool_call in message.tool_calls %}
|
| 54 |
+
{%- if tool_call.function %}
|
| 55 |
+
{%- set tool_call = tool_call.function %}
|
| 56 |
+
{%- endif %}
|
| 57 |
+
{{- '\n<tool_call>\n{"name": "' + tool_call.name + '", "arguments": ' + (tool_call.arguments if tool_call.arguments is string else tool_call.arguments | tojson) + '}\n</tool_call>' }}
|
| 58 |
+
{%- endfor %}
|
| 59 |
+
{%- endif %}
|
| 60 |
+
{{- '<|im_end|>\n' }}
|
| 61 |
+
{%- elif message['role'] == 'tool' %}
|
| 62 |
+
{%- if loop.index0 == 0 or messages[loop.index0 - 1]['role'] != 'tool' %}
|
| 63 |
+
{{- '<|im_start|>user\n' }}
|
| 64 |
+
{%- endif %}
|
| 65 |
+
{{- '<|function_output|>' + message['content'] }}
|
| 66 |
+
{%- if loop.last or messages[loop.index0 + 1]['role'] != 'tool' %}
|
| 67 |
+
{{- '<|im_end|>\n' }}
|
| 68 |
+
{%- endif %}
|
| 69 |
+
{%- endif %}
|
| 70 |
+
{%- endfor %}
|
| 71 |
+
|
| 72 |
+
{%- if add_generation_prompt %}
|
| 73 |
+
{{- '<|im_start|>assistant\n' }}
|
| 74 |
+
{%- if enable_thinking %}
|
| 75 |
+
{{- '<think>\n' }}
|
| 76 |
+
{%- endif %}
|
| 77 |
+
{%- endif %}
|
models/templates/ByteDance-Seed-OSS.jinja
ADDED
|
@@ -0,0 +1,171 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{# ----------‑‑‑ special token variables ‑‑‑---------- #}
|
| 2 |
+
{%- set bos_token = '<seed:bos>' -%}
|
| 3 |
+
{%- set eos_token = '<seed:eos>' -%}
|
| 4 |
+
{%- set pad_token = '<seed:pad>' -%}
|
| 5 |
+
{%- set toolcall_begin_token = '<seed:tool_call>' -%}
|
| 6 |
+
{%- set toolcall_end_token = '</seed:tool_call>' -%}
|
| 7 |
+
{%- set think_begin_token = '<seed:think>' -%}
|
| 8 |
+
{%- set think_end_token = '</seed:think>' -%}
|
| 9 |
+
{%- set budget_begin_token = '<seed:cot_budget_reflect>'-%}
|
| 10 |
+
{%- set budget_end_token = '</seed:cot_budget_reflect>'-%}
|
| 11 |
+
{# -------------- reflection-interval lookup -------------- #}
|
| 12 |
+
{%- if not thinking_budget is defined %}
|
| 13 |
+
{%- set thinking_budget = -1 -%}
|
| 14 |
+
{%- endif -%}
|
| 15 |
+
{%- set budget_reflections_v05 = {
|
| 16 |
+
0: 0,
|
| 17 |
+
512: 128,
|
| 18 |
+
1024: 256,
|
| 19 |
+
2048: 512,
|
| 20 |
+
4096: 512,
|
| 21 |
+
8192: 1024,
|
| 22 |
+
16384: 1024
|
| 23 |
+
} -%}
|
| 24 |
+
{# Find the first gear that is greater than or equal to the thinking_budget. #}
|
| 25 |
+
{%- set ns = namespace(interval = None) -%}
|
| 26 |
+
{%- for k, v in budget_reflections_v05 | dictsort -%}
|
| 27 |
+
{%- if ns.interval is none and thinking_budget <= k -%}
|
| 28 |
+
{%- set ns.interval = v -%}
|
| 29 |
+
{%- endif -%}
|
| 30 |
+
{%- endfor -%}
|
| 31 |
+
{# If it exceeds the maximum gear, use the value of the last gear #}
|
| 32 |
+
{%- if ns.interval is none -%}
|
| 33 |
+
{%- set ns.interval = budget_reflections_v05[16384] -%}
|
| 34 |
+
{%- endif -%}
|
| 35 |
+
{# ---------- Preprocess the system message ---------- #}
|
| 36 |
+
{%- if messages[0]["role"] == "system" %}
|
| 37 |
+
{%- set system_message = messages[0]["content"] %}
|
| 38 |
+
{%- set loop_messages = messages[1:] %}
|
| 39 |
+
{%- else %}
|
| 40 |
+
{%- set loop_messages = messages %}
|
| 41 |
+
{%- endif %}
|
| 42 |
+
{# ---------- Ensure tools exist ---------- #}
|
| 43 |
+
{%- if not tools is defined or tools is none %}
|
| 44 |
+
{%- set tools = [] %}
|
| 45 |
+
{%- endif %}
|
| 46 |
+
{# tools2doc.jinja #}
|
| 47 |
+
{%- macro py_type(t) -%}
|
| 48 |
+
{%- if t == "string" -%}str
|
| 49 |
+
{%- elif t in ("number", "integer") -%}int
|
| 50 |
+
{%- elif t == "boolean" -%}bool
|
| 51 |
+
{%- elif t == "array" -%}list
|
| 52 |
+
{%- else -%}Any{%- endif -%}
|
| 53 |
+
{%- endmacro -%}
|
| 54 |
+
{# ---------- Output the system block ---------- #}
|
| 55 |
+
{%- if system_message is defined %}
|
| 56 |
+
{{ bos_token + "system\n" + system_message }}
|
| 57 |
+
{%- else %}
|
| 58 |
+
{%- if tools is iterable and tools | length > 0 %}
|
| 59 |
+
{{ bos_token + "system\nYou are Doubao, a helpful AI assistant. You may call one or more functions to assist with the user query." }}
|
| 60 |
+
{%- endif %}
|
| 61 |
+
{%- endif %}
|
| 62 |
+
{%- if use_json_tooldef is defined and use_json_tooldef %}
|
| 63 |
+
|
| 64 |
+
{{"Tool List:\nYou are authorized to use the following tools (described in JSON Schema format). Before performing any task, you must decide how to call them based on the descriptions and parameters of these tools."}}
|
| 65 |
+
{{ tools | tojson(ensure_ascii=False) }}
|
| 66 |
+
{%- else %}
|
| 67 |
+
{%- for item in tools if item.type == "function" %}
|
| 68 |
+
|
| 69 |
+
|
| 70 |
+
Function:
|
| 71 |
+
def {{ item.function.name }}(
|
| 72 |
+
{%- for name, spec in item.function.parameters.properties.items() %}
|
| 73 |
+
{{- name }}: {{ py_type(spec.type) }}{% if not loop.last %},{% endif %}
|
| 74 |
+
{%- endfor %}):
|
| 75 |
+
"""
|
| 76 |
+
{{ item.function.description | trim }}
|
| 77 |
+
|
| 78 |
+
{# ---------- Args ---------- #}
|
| 79 |
+
{%- if item.function.parameters.properties %}
|
| 80 |
+
Args:
|
| 81 |
+
{%- for name, spec in item.function.parameters.properties.items() %}
|
| 82 |
+
|
| 83 |
+
- {{ name }} ({{ py_type(spec.type) }})
|
| 84 |
+
{%- if name in item.function.parameters.required %} [必填]{% else %} [选填]{% endif %}:
|
| 85 |
+
{{- " " ~ (spec.description or "") }}
|
| 86 |
+
{%- endfor %}
|
| 87 |
+
{%- endif %}
|
| 88 |
+
|
| 89 |
+
{# ---------- Returns ---------- #}
|
| 90 |
+
{%- if item.function.returns is defined
|
| 91 |
+
and item.function.returns.properties is defined
|
| 92 |
+
and item.function.returns.properties %}
|
| 93 |
+
Returns:
|
| 94 |
+
{%- for name, spec in item.function.returns.properties.items() %}
|
| 95 |
+
|
| 96 |
+
- {{ name }} ({{ py_type(spec.type) }}):
|
| 97 |
+
{{- " " ~ (spec.description or "") }}
|
| 98 |
+
{%- endfor %}
|
| 99 |
+
{%- endif %}
|
| 100 |
+
|
| 101 |
+
"""
|
| 102 |
+
{%- endfor %}
|
| 103 |
+
{%- endif %}
|
| 104 |
+
{%- if tools is iterable and tools | length > 0 %}
|
| 105 |
+
|
| 106 |
+
{{"工具调用请遵循如下格式:\n<seed:tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>value_1</parameter>\n<parameter=example_parameter_2>This is the value for the second parameter\nthat can span\nmultiple lines</parameter>\n</function>\n</seed:tool_call>\n"}}
|
| 107 |
+
{%- endif %}
|
| 108 |
+
{# End the system block line #}
|
| 109 |
+
{%- if system_message is defined or tools is iterable and tools | length > 0 %}
|
| 110 |
+
{{ eos_token }}
|
| 111 |
+
{%- endif %}
|
| 112 |
+
{# ---------- Thinking Budget ---------- #}
|
| 113 |
+
{%- if thinking_budget is defined %}
|
| 114 |
+
{%- if thinking_budget == 0 %}
|
| 115 |
+
{{ bos_token+"system" }}
|
| 116 |
+
{{ "You are an intelligent assistant that can answer questions in one step without the need for reasoning and thinking, that is, your thinking budget is 0. Next, please skip the thinking process and directly start answering the user's questions." }}
|
| 117 |
+
{{ eos_token }}
|
| 118 |
+
{%- elif not thinking_budget == -1 %}
|
| 119 |
+
{{ bos_token+"system" }}
|
| 120 |
+
{{ "You are an intelligent assistant with reflective ability. In the process of thinking and reasoning, you need to strictly follow the thinking budget, which is "}}{{thinking_budget}}{{". That is, you need to complete your thinking within "}}{{thinking_budget}}{{" tokens and start answering the user's questions. You will reflect on your thinking process every "}}{{ns.interval}}{{" tokens, stating how many tokens have been used and how many are left."}}
|
| 121 |
+
{{ eos_token }}
|
| 122 |
+
{%- endif %}
|
| 123 |
+
{%- endif %}
|
| 124 |
+
{# ---------- List the historical messages one by one ---------- #}
|
| 125 |
+
{%- for message in loop_messages %}
|
| 126 |
+
{%- if message.role == "assistant"
|
| 127 |
+
and message.tool_calls is defined
|
| 128 |
+
and message.tool_calls is iterable
|
| 129 |
+
and message.tool_calls | length > 0 %}
|
| 130 |
+
{{ bos_token + message.role }}
|
| 131 |
+
{%- if message.reasoning_content is defined and message.reasoning_content is string and message.reasoning_content | trim | length > 0 %}
|
| 132 |
+
{{ "\n" + think_begin_token + message.reasoning_content | trim + think_end_token }}
|
| 133 |
+
{%- endif %}
|
| 134 |
+
{%- if message.content is defined and message.content is string and message.content | trim | length > 0 %}
|
| 135 |
+
{{ "\n" + message.content | trim + "\n" }}
|
| 136 |
+
{%- endif %}
|
| 137 |
+
{%- for tool_call in message.tool_calls %}
|
| 138 |
+
{%- if tool_call.function is defined %}{% set tool_call = tool_call.function %}{% endif %}
|
| 139 |
+
{{ "\n" + toolcall_begin_token + "\n<function=" + tool_call.name + ">\n" }}
|
| 140 |
+
{%- if tool_call.arguments is defined %}
|
| 141 |
+
{%- for arg_name, arg_value in tool_call.arguments | items %}
|
| 142 |
+
{{ "<parameter=" + arg_name + ">" }}
|
| 143 |
+
{%- set arg_value = arg_value if arg_value is string else arg_value | string %}
|
| 144 |
+
{{ arg_value+"</parameter>\n" }}
|
| 145 |
+
{%- endfor %}
|
| 146 |
+
{%- endif %}
|
| 147 |
+
{{ "</function>\n" + toolcall_end_token }}
|
| 148 |
+
{%- endfor %}
|
| 149 |
+
{{ eos_token }}
|
| 150 |
+
{%- elif message.role in ["user", "system"] %}
|
| 151 |
+
{{ bos_token + message.role + "\n" + message.content + eos_token }}
|
| 152 |
+
{%- elif message.role == "assistant" %}
|
| 153 |
+
{{ bos_token + message.role }}
|
| 154 |
+
{%- if message.reasoning_content is defined and message.reasoning_content is string and message.reasoning_content | trim | length > 0 %}
|
| 155 |
+
{{ "\n" + think_begin_token + message.reasoning_content | trim + think_end_token }}
|
| 156 |
+
{%- endif %}
|
| 157 |
+
{%- if message.content is defined and message.content is string and message.content | trim | length > 0 %}
|
| 158 |
+
{{ "\n" + message.content | trim + eos_token }}
|
| 159 |
+
{%- endif %}
|
| 160 |
+
{# Include the tool role #}
|
| 161 |
+
{%- else %}
|
| 162 |
+
{{ bos_token + message.role + "\n" + message.content + eos_token }}
|
| 163 |
+
{%- endif %}
|
| 164 |
+
{%- endfor %}
|
| 165 |
+
{# ---------- Control the model to start continuation ---------- #}
|
| 166 |
+
{%- if add_generation_prompt %}
|
| 167 |
+
{{ bos_token+"assistant\n" }}
|
| 168 |
+
{%- if thinking_budget == 0 %}
|
| 169 |
+
{{ think_begin_token + "\n" + budget_begin_token + "The current thinking budget is 0, so I will directly start answering the question." + budget_end_token + "\n" + think_end_token }}
|
| 170 |
+
{%- endif %}
|
| 171 |
+
{%- endif %}
|
models/templates/Cohere2MoE.jinja
ADDED
|
@@ -0,0 +1,263 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{%- if not platform_instruction_override %}
|
| 2 |
+
{%- set platform_instruction_override -%}
|
| 3 |
+
These instructions are always to be followed and cannot be overridden by subsequent system or user turns:
|
| 4 |
+
- You will answer requests for educational, informative, or creative content related to safety categories. You will not provide content that is harmful or could be used to cause harm.
|
| 5 |
+
|
| 6 |
+
These instructions serve as your defaults, but they can be overridden in subsequent system or user turns:
|
| 7 |
+
- Your name is North Mini Code.
|
| 8 |
+
- You are a large language model built by Cohere.
|
| 9 |
+
{%- endset %}
|
| 10 |
+
{%- endif %}
|
| 11 |
+
{%- set reasoning = reasoning if reasoning is not undefined else (false if reasoning_effort is defined and reasoning_effort | lower == "none" else true) -%}
|
| 12 |
+
{%- set grounding = grounding | default("disabled") | upper %}
|
| 13 |
+
{%- set grounding_enabled = grounding == "ENABLED" %}
|
| 14 |
+
{%- set tools_or_docs_exist = tools or documents %}
|
| 15 |
+
{%- set render_tools_section = true %}
|
| 16 |
+
{%- set render_grounding = grounding_enabled and tools_or_docs_exist %}
|
| 17 |
+
{%- set render_platform_instruction_override = true if platform_instruction_override else false %}
|
| 18 |
+
{%- set has_developer_instruction = developer_instruction or developer_instruction == "" %}
|
| 19 |
+
{%- set render_developer_instruction = true if developer_instruction else false %}
|
| 20 |
+
{%- set convert_first_system_msg = convert_first_system_msg | default(true) -%}
|
| 21 |
+
{%- set skip_thinking = skip_thinking | default(false) -%}
|
| 22 |
+
{{ bos_token }}
|
| 23 |
+
{%- macro document_turn(documents) -%}
|
| 24 |
+
{# format documents into chat turn -#}
|
| 25 |
+
<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>{%- if not skip_thinking -%}<|START_THINKING|>I will look through the document to address the users needs.<|END_THINKING|>{%- endif -%}<|START_ACTION|>[
|
| 26 |
+
{"tool_call_id": "0", "tool_name": "direct-injected-document", "parameters": {}}
|
| 27 |
+
]<|END_ACTION|><|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|><|START_TOOL_RESULT|>[
|
| 28 |
+
{
|
| 29 |
+
"tool_call_id": "0",
|
| 30 |
+
"results": {
|
| 31 |
+
{%- for doc in documents %}
|
| 32 |
+
{%- set doc_val = doc.data if doc.data else doc %}
|
| 33 |
+
|
| 34 |
+
"{{ loop.index0 }}": {{ doc_val|tojson }}{% if not loop.last %},
|
| 35 |
+
{%- endif %}
|
| 36 |
+
{%- endfor %}
|
| 37 |
+
|
| 38 |
+
},
|
| 39 |
+
"is_error": null
|
| 40 |
+
}
|
| 41 |
+
]<|END_TOOL_RESULT|><|END_OF_TURN_TOKEN|>{%- endmacro %}
|
| 42 |
+
{%- macro tool_call_id_to_int(messages, tool_call_id) %}
|
| 43 |
+
{%- if regen_tool_call_ids -%}
|
| 44 |
+
{%- set counter = namespace(value=0) %}
|
| 45 |
+
{%- set tool_call_id_seen = namespace(value=false) %}
|
| 46 |
+
{%- for msg in messages %}
|
| 47 |
+
{%- if msg.tool_calls %}
|
| 48 |
+
{%- for tool_call in msg.tool_calls %}
|
| 49 |
+
{%- if tool_call.id == tool_call_id and not tool_call_id_seen.value -%}
|
| 50 |
+
{{ counter.value }}
|
| 51 |
+
{%- set tool_call_id_seen.value = true %}
|
| 52 |
+
{%- endif %}
|
| 53 |
+
{%- set counter.value = counter.value + 1 %}
|
| 54 |
+
{%- endfor %}
|
| 55 |
+
{%- endif %}
|
| 56 |
+
{%- endfor %}
|
| 57 |
+
{%- else -%}
|
| 58 |
+
{{ tool_call_id }}
|
| 59 |
+
{%- endif -%}
|
| 60 |
+
{%- endmacro %}
|
| 61 |
+
{%- macro format_tool_message(messages, tool_msg) -%}
|
| 62 |
+
{#- format tool message #}{
|
| 63 |
+
"tool_call_id": "{{ tool_call_id_to_int(messages, tool_msg.tool_call_id) }}",
|
| 64 |
+
"results": {
|
| 65 |
+
{%- if tool_msg.content is mapping or tool_msg.content is string %}
|
| 66 |
+
|
| 67 |
+
{% if tool_msg.content is string -%}
|
| 68 |
+
{%- set text_wrapper = {"content": tool_msg.content} -%}
|
| 69 |
+
{%- else -%}
|
| 70 |
+
{%- set text_wrapper = tool_msg.content -%}
|
| 71 |
+
{%- endif %}
|
| 72 |
+
"0": {{ text_wrapper|tojson }}
|
| 73 |
+
{%- else %}
|
| 74 |
+
{%- for content in tool_msg.content %}
|
| 75 |
+
|
| 76 |
+
"{{ loop.index0 }}": {{ print_tool_content(content) }}{% if not loop.last %},{% endif %}
|
| 77 |
+
{%- endfor %}
|
| 78 |
+
{%- endif %}
|
| 79 |
+
|
| 80 |
+
},
|
| 81 |
+
"is_error": null
|
| 82 |
+
}
|
| 83 |
+
{%- endmacro -%}
|
| 84 |
+
{%- macro print_tool_content(item) %}
|
| 85 |
+
{%- if item.type|lower == "text" -%}
|
| 86 |
+
{%- set text_wrapper = {"content": item.text} -%}
|
| 87 |
+
{{ text_wrapper|tojson }}
|
| 88 |
+
{%- elif item.type|lower == "document" and item.document and "data" in item.document -%}
|
| 89 |
+
{{ item.document.data|tojson }}
|
| 90 |
+
{%- else -%}
|
| 91 |
+
{{ item|tojson }}
|
| 92 |
+
{%- endif -%}
|
| 93 |
+
{%- endmacro %}
|
| 94 |
+
{%- macro print_msg(msg) %}
|
| 95 |
+
{%- if msg is string -%}
|
| 96 |
+
<|START_TEXT|>{{ msg }}<|END_TEXT|>
|
| 97 |
+
{%- elif msg.content is string -%}
|
| 98 |
+
<|START_TEXT|>{{ msg.content }}<|END_TEXT|>
|
| 99 |
+
{%- else %}
|
| 100 |
+
{%- set last_was_text = namespace(value=false) %}
|
| 101 |
+
{%- for content in msg.content %}
|
| 102 |
+
{%- if content.type|lower == "text" -%}
|
| 103 |
+
{%- if not last_was_text.value -%}
|
| 104 |
+
<|START_TEXT|>
|
| 105 |
+
{%- endif -%}
|
| 106 |
+
{{ content.text }}
|
| 107 |
+
{%- if loop.last -%}
|
| 108 |
+
<|END_TEXT|>
|
| 109 |
+
{%- endif %}
|
| 110 |
+
{%- set last_was_text.value = true -%}
|
| 111 |
+
{%- else -%}
|
| 112 |
+
{%- if last_was_text.value -%}
|
| 113 |
+
<|END_TEXT|>
|
| 114 |
+
{%- endif -%}
|
| 115 |
+
{%- set last_was_text.value = false -%}
|
| 116 |
+
{%- endif -%}
|
| 117 |
+
{%- if content.type|lower == "image" -%}
|
| 118 |
+
{%- if content.data -%}
|
| 119 |
+
{{ content.data }}
|
| 120 |
+
{%- else -%}
|
| 121 |
+
<|IMG_PATCH|>
|
| 122 |
+
{%- endif -%}
|
| 123 |
+
{%- endif -%}
|
| 124 |
+
{%- endfor %}
|
| 125 |
+
{%- endif %}
|
| 126 |
+
{%- endmacro %}
|
| 127 |
+
{%- macro print_thinking(msg) %}
|
| 128 |
+
{%- if msg.reasoning -%}
|
| 129 |
+
{{ msg.reasoning }}
|
| 130 |
+
{%- elif msg.reasoning_content -%}
|
| 131 |
+
{{ msg.reasoning_content }}
|
| 132 |
+
{%- elif msg.thinking -%}
|
| 133 |
+
{{ msg.thinking }}
|
| 134 |
+
{%- elif msg.content and msg.content[0].thinking -%}
|
| 135 |
+
{{ msg.content[0].thinking }}
|
| 136 |
+
{%- endif %}
|
| 137 |
+
{%- endmacro %}
|
| 138 |
+
{%- if messages and messages[0]['role']|lower == 'system' and not has_developer_instruction and convert_first_system_msg %}{%- set developer_instruction = messages[0] %}{%- set render_developer_instruction = true %}{%- set initial_instruction_message = true %}{% endif %}
|
| 139 |
+
{%- set json_object = true if response_format and response_format.type == "json_object" else false %}
|
| 140 |
+
{%- set json_schema = (response_format.json_schema or response_format.schema) if response_format %}
|
| 141 |
+
{%- set json_mode = json_object or json_schema %}
|
| 142 |
+
{%- set tool_idx = namespace(value=0) %}
|
| 143 |
+
{%- set tool_ids_seen = namespace(value=[]) %}
|
| 144 |
+
{%- set regen_tool_call_ids = regen_tool_call_ids | default(true) -%}
|
| 145 |
+
{%- set sent_documents = namespace(value=false) -%}
|
| 146 |
+
|
| 147 |
+
{%- if render_tools_section or render_platform_instruction_override or render_grounding or json_mode -%}
|
| 148 |
+
<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|><|START_TEXT|>
|
| 149 |
+
{%- elif not render_developer_instruction -%}
|
| 150 |
+
<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>
|
| 151 |
+
{%- endif %}
|
| 152 |
+
|
| 153 |
+
{%- set rendered_platform_turn_chunk = false %}
|
| 154 |
+
|
| 155 |
+
{%- if render_platform_instruction_override -%}
|
| 156 |
+
{{ platform_instruction_override }}
|
| 157 |
+
{% set rendered_platform_turn_chunk = true %}
|
| 158 |
+
{%- else %}
|
| 159 |
+
{%- endif %}
|
| 160 |
+
|
| 161 |
+
{%- if render_grounding -%}
|
| 162 |
+
{%- if rendered_platform_turn_chunk %}
|
| 163 |
+
|
| 164 |
+
{% endif -%}
|
| 165 |
+
Note that both your responses and reflections can be grounded. Grounding means you associate pieces of texts (called "spans") with those specific tool results that support them (called "sources"). And you use a pair of tags "<co>" and "</co>" to indicate when a span can be grounded onto a list of sources, listing them out in the closing tag. Sources from the same tool call are grouped together and listed as "{tool_call_id}:[{list of result indices}]", before they are joined together by ",". E.g., "<co>span</co: 0:[1,2],1:[0]>" means that "span" is supported by result 1 and 2 from "tool_call_id=0" as well as result 0 from "tool_call_id=1".
|
| 166 |
+
{% set rendered_platform_turn_chunk = true %}
|
| 167 |
+
{%- endif %}
|
| 168 |
+
|
| 169 |
+
{%- if render_tools_section %}
|
| 170 |
+
{%- if rendered_platform_turn_chunk %}
|
| 171 |
+
|
| 172 |
+
{% endif %}
|
| 173 |
+
# Available Tools
|
| 174 |
+
```json
|
| 175 |
+
[
|
| 176 |
+
{% if tools_or_docs_exist %}
|
| 177 |
+
{%- if documents %}
|
| 178 |
+
{"name": "direct-injected-document", "description": "This is a special tool to directly inject user-uploaded documents into the chat as additional context. DO NOT use this tool by yourself!", "parameters": {"type": "object", "properties": {}, "required": []}, "responses": {"200": {"description": "Successfully returned a list of chunked text snippets from the directly uploaded documents.", "content": {"application/json": {"schema": {"type": "array", "items": {"type": "object", "required": ["url", "snippet"], "properties": {"url": {"type": "string", "description": "The url of the uploaded document."}, "snippet": {"type": "string", "description": "The text snippet for the returned document chunk."}}}}}}}}}
|
| 179 |
+
{%- if tools %},
|
| 180 |
+
{% else %}
|
| 181 |
+
|
| 182 |
+
{% endif %}
|
| 183 |
+
{%- endif %}
|
| 184 |
+
{%- for tool in tools %}
|
| 185 |
+
{"name": "{{ tool['function']['name'] }}", "description": "{{ tool['function']['description'] }}", "parameters": {{ tool['function']['parameters']|tojson }}, "responses": null}
|
| 186 |
+
{%- if not loop.last %},{% endif %}
|
| 187 |
+
|
| 188 |
+
{% endfor %}
|
| 189 |
+
{%- else %}
|
| 190 |
+
|
| 191 |
+
{% endif %}
|
| 192 |
+
]
|
| 193 |
+
```
|
| 194 |
+
{%- set rendered_platform_turn_chunk = true %}
|
| 195 |
+
{%- endif -%}
|
| 196 |
+
|
| 197 |
+
{%- if json_mode -%}
|
| 198 |
+
{%- if rendered_platform_turn_chunk %}
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
{% endif -%}
|
| 202 |
+
When generating JSON objects, do not generate block markers. Generate an object directly without prefixing with ```json. Return only the JSON and nothing else.
|
| 203 |
+
{%- if json_schema %}
|
| 204 |
+
|
| 205 |
+
Your output should adhere to the following json schema:
|
| 206 |
+
{{ json_schema }}
|
| 207 |
+
{%- endif -%}
|
| 208 |
+
{%- set rendered_platform_turn_chunk = true %}
|
| 209 |
+
{%- endif %}
|
| 210 |
+
{%- if rendered_platform_turn_chunk -%}
|
| 211 |
+
<|END_TEXT|><|END_OF_TURN_TOKEN|>
|
| 212 |
+
{%- elif not render_developer_instruction -%}
|
| 213 |
+
<|END_OF_TURN_TOKEN|>
|
| 214 |
+
{%- endif %}
|
| 215 |
+
{%- if render_developer_instruction -%}
|
| 216 |
+
<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>{{ print_msg(developer_instruction) }}<|END_OF_TURN_TOKEN|>
|
| 217 |
+
{%- endif %}
|
| 218 |
+
{%- for message in messages %}
|
| 219 |
+
{%- set msg_role_downcased = message.role | lower %}
|
| 220 |
+
{%- if msg_role_downcased == 'system' and (not (loop.first and initial_instruction_message)) -%}
|
| 221 |
+
<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>{{ print_msg(message) }}<|END_OF_TURN_TOKEN|>
|
| 222 |
+
{%- elif msg_role_downcased == 'user' -%}
|
| 223 |
+
<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{{ print_msg(message) }}<|END_OF_TURN_TOKEN|>
|
| 224 |
+
{%- if documents and not sent_documents.value %}{%- set sent_documents.value = true %}{% set tool_idx.value = tool_idx.value + 1 %}{{ document_turn(documents) }}{% endif %}
|
| 225 |
+
{%- elif msg_role_downcased == 'assistant' or msg_role_downcased == 'chatbot' -%}
|
| 226 |
+
<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>
|
| 227 |
+
{%- if message.tool_calls %}
|
| 228 |
+
{% if not skip_thinking %}
|
| 229 |
+
{% if message.tool_plan -%}
|
| 230 |
+
<|START_THINKING|>{{ message.tool_plan }}<|END_THINKING|>
|
| 231 |
+
{%- elif message.reasoning or message.reasoning_content or message.thinking or (message.content and message.content[0].type == "thinking") -%}
|
| 232 |
+
<|START_THINKING|>{{ print_thinking(message) }}<|END_THINKING|>
|
| 233 |
+
{%- endif %}
|
| 234 |
+
{%- endif %}<|START_ACTION|>[
|
| 235 |
+
{%- for tc in message.tool_calls %}
|
| 236 |
+
|
| 237 |
+
{"tool_call_id": "{%- if regen_tool_call_ids -%}{{ tool_idx.value }}{%- else -%}{{ tc.id }}{%- endif -%}", "tool_name": "{{ tc['function']['name'] }}", "parameters": {{ tc['function']['arguments']|tojson }}}{% if not loop.last %},{% endif %}
|
| 238 |
+
{%- set tool_idx.value = tool_idx.value + 1 %}
|
| 239 |
+
{%- endfor %}
|
| 240 |
+
|
| 241 |
+
]<|END_ACTION|><|END_OF_TURN_TOKEN|>
|
| 242 |
+
{%- else -%}
|
| 243 |
+
{% if (message.reasoning or message.reasoning_content or message.thinking or (message.content and message.content[0].type == "thinking")) and not skip_thinking -%}
|
| 244 |
+
<|START_THINKING|>{{ print_thinking(message) }}<|END_THINKING|>
|
| 245 |
+
{%- endif -%}
|
| 246 |
+
{{ print_msg(message) }}<|END_OF_TURN_TOKEN|>
|
| 247 |
+
{%- endif %}
|
| 248 |
+
{%- elif msg_role_downcased == 'tool' and message.tool_call_id not in tool_ids_seen.value -%}
|
| 249 |
+
<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|><|START_TOOL_RESULT|>[
|
| 250 |
+
{{ format_tool_message(messages, message) }}
|
| 251 |
+
{%- for msg in messages[loop.index0 + 1:] %}
|
| 252 |
+
|
| 253 |
+
{%- if msg.role | lower == 'tool' %},
|
| 254 |
+
{{ format_tool_message(messages, msg) }}
|
| 255 |
+
{%- set tool_ids_seen.value = tool_ids_seen.value + [msg.tool_call_id] %}
|
| 256 |
+
{%- else %}
|
| 257 |
+
{%- break %}
|
| 258 |
+
{%- endif %}
|
| 259 |
+
{%- endfor %}
|
| 260 |
+
|
| 261 |
+
]<|END_TOOL_RESULT|><|END_OF_TURN_TOKEN|>
|
| 262 |
+
{%- endif %}
|
| 263 |
+
{%- endfor %}{%- if add_generation_prompt -%}<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>{% if reasoning %}<|START_THINKING|>{% else %}<|START_THINKING|><|END_THINKING|>{% endif %}{%- endif %}
|
models/templates/CohereForAI-c4ai-command-r-plus-tool_use.jinja
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
|
| 2 |
+
{%- macro json_to_python_type(json_spec) %}
|
| 3 |
+
{%- set basic_type_map = {
|
| 4 |
+
"string": "str",
|
| 5 |
+
"number": "float",
|
| 6 |
+
"integer": "int",
|
| 7 |
+
"boolean": "bool"
|
| 8 |
+
} %}
|
| 9 |
+
|
| 10 |
+
{%- if basic_type_map[json_spec.type] is defined %}
|
| 11 |
+
{{- basic_type_map[json_spec.type] }}
|
| 12 |
+
{%- elif json_spec.type == "array" %}
|
| 13 |
+
{{- "List[" + json_to_python_type(json_spec.items) + "]"}}
|
| 14 |
+
{%- elif json_spec.type == "object" %}
|
| 15 |
+
{{- "Dict[str, " + json_to_python_type(json_spec.additionalProperties) + ']'}}
|
| 16 |
+
{%- elif json_spec.type is iterable %}
|
| 17 |
+
{{- "Union[" }}
|
| 18 |
+
{%- for t in json_spec.type %}
|
| 19 |
+
{{- json_to_python_type({"type": t}) }}
|
| 20 |
+
{%- if not loop.last %}
|
| 21 |
+
{{- "," }}
|
| 22 |
+
{%- endif %}
|
| 23 |
+
{%- endfor %}
|
| 24 |
+
{{- "]" }}
|
| 25 |
+
{%- else %}
|
| 26 |
+
{{- "Any" }}
|
| 27 |
+
{%- endif %}
|
| 28 |
+
{%- endmacro %}
|
| 29 |
+
|
| 30 |
+
{%- macro old_tool_parser(tools) %}
|
| 31 |
+
{%- for tool in tools %}
|
| 32 |
+
{%- if loop.index0 != 0 %}
|
| 33 |
+
{{- '\n\n' }}
|
| 34 |
+
{%- endif %}
|
| 35 |
+
{{- '```python\ndef ' + tool.name + '(' }}
|
| 36 |
+
{%- for param_name, param_fields in tool.parameter_definitions|items %}
|
| 37 |
+
{%- if loop.index0 != 0 %}
|
| 38 |
+
{{- ', '}}
|
| 39 |
+
{%- endif %}
|
| 40 |
+
{{- param_name + ': ' }}
|
| 41 |
+
{%- if not param_fields.required %}
|
| 42 |
+
{{- 'Optional[' + param_fields.type + '] = None'}}
|
| 43 |
+
{%- else %}
|
| 44 |
+
{{- param_fields.type }}
|
| 45 |
+
{%- endif %}
|
| 46 |
+
{%- endfor %}
|
| 47 |
+
{{- ') -> List[Dict]:\n """'}}
|
| 48 |
+
{{- tool.description }}
|
| 49 |
+
{%- if tool.parameter_definitions|length != 0 %}
|
| 50 |
+
{{- '\n\n Args:\n '}}
|
| 51 |
+
{%- for param_name, param_fields in tool.parameter_definitions|items %}
|
| 52 |
+
{%- if loop.index0 != 0 %}
|
| 53 |
+
{{- '\n ' }}
|
| 54 |
+
{%- endif %}
|
| 55 |
+
{{- param_name + ' ('}}
|
| 56 |
+
{%- if not param_fields.required %}
|
| 57 |
+
{{- 'Optional[' + param_fields.type + ']'}}
|
| 58 |
+
{%- else %}
|
| 59 |
+
{{- param_fields.type }}
|
| 60 |
+
{%- endif %}
|
| 61 |
+
{{- '): ' + param_fields.description }}
|
| 62 |
+
{%- endfor %}
|
| 63 |
+
{%- endif %}
|
| 64 |
+
{{- '\n """\n pass\n```' }}
|
| 65 |
+
{%- endfor %}
|
| 66 |
+
{%- endmacro %}
|
| 67 |
+
|
| 68 |
+
{%- macro new_tool_parser(tools) %}
|
| 69 |
+
{%- for tool in tools %}
|
| 70 |
+
{%- if loop.index0 != 0 %}
|
| 71 |
+
{{- '\n\n'}}
|
| 72 |
+
{%- endif %}
|
| 73 |
+
{%- if tool.function is defined %}
|
| 74 |
+
{%- set tool = tool.function %}
|
| 75 |
+
{%- endif %}
|
| 76 |
+
{{-'```python
|
| 77 |
+
def ' + tool.name + '('}}
|
| 78 |
+
{%- for param_name, param_fields in tool.parameters.properties|items %}
|
| 79 |
+
{%- if loop.index0 != 0 %}
|
| 80 |
+
{{- ', '}}
|
| 81 |
+
{%- endif %}
|
| 82 |
+
{{-param_name + ": "}}
|
| 83 |
+
{%- if not param_name in tool.parameters.required %}
|
| 84 |
+
{{-'Optional[' + json_to_python_type(param_fields) + '] = None'}}
|
| 85 |
+
{%- else %}
|
| 86 |
+
{{- json_to_python_type(param_fields) }}
|
| 87 |
+
{%- endif %}
|
| 88 |
+
{%- endfor %}
|
| 89 |
+
{{- ') -> List[Dict]:
|
| 90 |
+
"""'}}
|
| 91 |
+
{{- tool.description }}
|
| 92 |
+
{%- if tool.parameters.properties|length != 0 %}
|
| 93 |
+
{{- '\n\n Args:\n '}}
|
| 94 |
+
{%- for param_name, param_fields in tool.parameters.properties|items %}
|
| 95 |
+
{%- if loop.index0 != 0 %}
|
| 96 |
+
{{- '\n ' }}
|
| 97 |
+
{%- endif %}
|
| 98 |
+
{{- param_name + ' ('}}
|
| 99 |
+
{%- if not param_name in tool.parameters.required %}
|
| 100 |
+
{{-'Optional[' + json_to_python_type(param_fields) + ']'}}
|
| 101 |
+
{%- else %}
|
| 102 |
+
{{- json_to_python_type(param_fields) }}
|
| 103 |
+
{%- endif %}
|
| 104 |
+
{{- '): ' + param_fields.description }}
|
| 105 |
+
{%- endfor %}
|
| 106 |
+
{%- endif %}
|
| 107 |
+
{{- '\n """\n pass\n```' }}
|
| 108 |
+
{%- endfor %}
|
| 109 |
+
{%- endmacro %}
|
| 110 |
+
|
| 111 |
+
{{- bos_token }}
|
| 112 |
+
{%- if messages[0]['role'] == 'system' %}
|
| 113 |
+
{%- set loop_messages = messages[1:] %}
|
| 114 |
+
{%- set system_message = messages[0]['content'] %}
|
| 115 |
+
{%- else %}
|
| 116 |
+
{%- set loop_messages = messages %}
|
| 117 |
+
{%- set system_message = '## Task and Context\nYou help people answer their questions and other requests interactively. You will be asked a very wide array of requests on all kinds of topics. You will be equipped with a wide range of search engines or similar tools to help you, which you use to research your answer. You should focus on serving the user\'s needs as best you can, which will be wide-ranging.\n\n## Style Guide\nUnless the user asks for a different style of answer, you should answer in full sentences, using proper grammar and spelling.' %}
|
| 118 |
+
{%- endif %}
|
| 119 |
+
{{- '<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>' }}
|
| 120 |
+
{{- '# Safety Preamble' }}
|
| 121 |
+
{{- '
|
| 122 |
+
The instructions in this section override those in the task description and style guide sections. Don\'t answer questions that are harmful or immoral.' }}
|
| 123 |
+
{{- '
|
| 124 |
+
|
| 125 |
+
# System Preamble' }}
|
| 126 |
+
{{- '
|
| 127 |
+
## Basic Rules' }}
|
| 128 |
+
{{- '
|
| 129 |
+
You are a powerful conversational AI trained by Cohere to help people. You are augmented by a number of tools, and your job is to use and consume the output of these tools to best help the user. You will see a conversation history between yourself and a user, ending with an utterance from the user. You will then see a specific instruction instructing you what kind of response to generate. When you answer the user\'s requests, you cite your sources in your answers, according to those instructions.' }}
|
| 130 |
+
{{- '
|
| 131 |
+
|
| 132 |
+
# User Preamble' }}
|
| 133 |
+
{{- '
|
| 134 |
+
' + system_message }}
|
| 135 |
+
{{-'
|
| 136 |
+
|
| 137 |
+
## Available Tools
|
| 138 |
+
Here is a list of tools that you have available to you:
|
| 139 |
+
|
| 140 |
+
'}}
|
| 141 |
+
{%- set ns = namespace(new_tools=true) %}
|
| 142 |
+
{%- for tool in tools %}
|
| 143 |
+
{%- if tool.parameter_definitions is defined %}
|
| 144 |
+
{%- set ns.new_tools = false %}
|
| 145 |
+
{%- endif %}
|
| 146 |
+
{%- endfor %}
|
| 147 |
+
{%- if ns.new_tools %}
|
| 148 |
+
{{- new_tool_parser(tools) }}
|
| 149 |
+
{%- else %}
|
| 150 |
+
{{- old_tool_parser(tools) }}
|
| 151 |
+
{%- endif %}
|
| 152 |
+
{{- '<|END_OF_TURN_TOKEN|>'}}
|
| 153 |
+
{%- for message in loop_messages %}
|
| 154 |
+
{%- set content = message['content'] %}
|
| 155 |
+
{%- if message.role == 'user' %}
|
| 156 |
+
{{- '<|START_OF_TURN_TOKEN|><|USER_TOKEN|>' + content|trim + '<|END_OF_TURN_TOKEN|>' }}
|
| 157 |
+
{%- elif message.role == 'system' %}
|
| 158 |
+
{{- '<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>' + content|trim + '<|END_OF_TURN_TOKEN|>' }}
|
| 159 |
+
{%- elif message.role == 'assistant' and message.tool_calls is defined %}
|
| 160 |
+
{{- '<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>' }}
|
| 161 |
+
{%- if message.content is defined %}
|
| 162 |
+
{{- message.content|trim }}
|
| 163 |
+
{%- endif %}
|
| 164 |
+
{{- '\nAction:\n```json\n[\n' }}
|
| 165 |
+
{%- for tool_call in message.tool_calls %}
|
| 166 |
+
{%- if tool_call.function is defined %}
|
| 167 |
+
{%- set tool_call = tool_call.function %}
|
| 168 |
+
{%- endif %}
|
| 169 |
+
{{- '{\n'|indent(4, first=true) }}
|
| 170 |
+
{{- '"tool_name": "'|indent(8, first=true) + tool_call.name + '",\n' }}
|
| 171 |
+
{{- '"parameters": '|indent(8, first=true) }}
|
| 172 |
+
{%- if tool_call.arguments is defined and tool_call.arguments|length > 0 %}
|
| 173 |
+
{{- tool_call.arguments|tojson(indent=4)|indent(8) }}
|
| 174 |
+
{{- '\n' }}
|
| 175 |
+
{%- else %}
|
| 176 |
+
{{- '{}\n' }}
|
| 177 |
+
{%- endif %}
|
| 178 |
+
{{- '}'|indent(4, first=true) }}
|
| 179 |
+
{%- if not loop.last %}
|
| 180 |
+
{{- ',\n' }}
|
| 181 |
+
{%- endif %}
|
| 182 |
+
{%- endfor %}
|
| 183 |
+
{{- "\n]```\n" }}
|
| 184 |
+
{%- elif message.role == 'assistant' %}
|
| 185 |
+
{{- '<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>' + content|trim + '<|END_OF_TURN_TOKEN|>' }}
|
| 186 |
+
{%- elif message.role == 'tool' %}
|
| 187 |
+
{{- '<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|><results>\n' }}
|
| 188 |
+
{{- message.content|trim }}
|
| 189 |
+
{{- '</results><|END_OF_TURN_TOKEN|>' }}
|
| 190 |
+
{%- endif %}
|
| 191 |
+
{%- endfor %}
|
| 192 |
+
{{-'<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>Write \'Action:\' followed by a json-formatted list of actions that you want to perform in order to produce a good response to the user\'s last input. You can use any of the supplied tools any number of times, but you should aim to execute the minimum number of necessary actions for the input. You should use the `directly-answer` tool if calling the other tools is unnecessary. The list of actions you want to call should be formatted as a list of json objects, for example:
|
| 193 |
+
```json
|
| 194 |
+
[
|
| 195 |
+
{
|
| 196 |
+
"tool_name": title of the tool in the specification,
|
| 197 |
+
"parameters": a dict of parameters to input into the tool as they are defined in the specs, or {} if it takes no parameters
|
| 198 |
+
}
|
| 199 |
+
]```<|END_OF_TURN_TOKEN|>'}}
|
| 200 |
+
{%- if add_generation_prompt %}
|
| 201 |
+
{{- '<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>' }}
|
| 202 |
+
{%- endif %}
|
models/templates/CohereForAI-c4ai-command-r7b-12-2024-tool_use.jinja
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{{ bos_token }}{%- macro document_turn(documents) -%}
|
| 2 |
+
{# format documents into chat turn #}
|
| 3 |
+
<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|><|START_THINKING|>I will look through the document to address the users needs.<|END_THINKING|><|START_ACTION|>[
|
| 4 |
+
{"tool_call_id": "0", "tool_name": "direct-injected-document", "parameters": {}}
|
| 5 |
+
]<|END_ACTION|><|END_OF_TURN_TOKEN|><|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|><|START_TOOL_RESULT|>[
|
| 6 |
+
{
|
| 7 |
+
"tool_call_id": "0",
|
| 8 |
+
"results": {
|
| 9 |
+
{% for doc in documents %}
|
| 10 |
+
"{{ loop.index0 }}": {{doc|tojson}}{% if not loop.last %},
|
| 11 |
+
{% endif %}
|
| 12 |
+
{% endfor %}
|
| 13 |
+
|
| 14 |
+
},
|
| 15 |
+
"is_error": null
|
| 16 |
+
}
|
| 17 |
+
]<|END_TOOL_RESULT|><|END_OF_TURN_TOKEN|>{%- endmacro %}
|
| 18 |
+
{%- macro tool_call_id_to_int(messages, tool_call_id) %}
|
| 19 |
+
{%- set counter = namespace(value=0) %}
|
| 20 |
+
{%- set tool_call_id_seen = namespace(value=false) %}
|
| 21 |
+
{%- for msg in messages %}
|
| 22 |
+
{%- if msg.tool_calls %}
|
| 23 |
+
{%- for tool_call in msg.tool_calls %}
|
| 24 |
+
{%- if tool_call.id == tool_call_id and not tool_call_id_seen.value -%}
|
| 25 |
+
{{ counter.value }}
|
| 26 |
+
{%- set tool_call_id_seen.value = true %}
|
| 27 |
+
{%- endif %}
|
| 28 |
+
{%- set counter.value = counter.value + 1 %}
|
| 29 |
+
{%- endfor %}
|
| 30 |
+
{%- endif %}
|
| 31 |
+
{%- endfor %}
|
| 32 |
+
{%- endmacro %}
|
| 33 |
+
{%- macro format_tool_message(messages, tool_msg) -%}
|
| 34 |
+
{# format tool message #}
|
| 35 |
+
{
|
| 36 |
+
"tool_call_id": "{{ tool_call_id_to_int(messages, tool_msg.tool_call_id) }}",
|
| 37 |
+
"results": {
|
| 38 |
+
"0": {{ tool_msg.content|tojson }}
|
| 39 |
+
},
|
| 40 |
+
"is_error": null
|
| 41 |
+
}
|
| 42 |
+
{%- endmacro -%}
|
| 43 |
+
{%- if messages and messages[0]['role']|lower == 'system' %}{%- set developer_preamble = messages[0]['content'] %}{% endif %}
|
| 44 |
+
{%- set tool_idx = namespace(value=0) %}
|
| 45 |
+
{%- set tool_ids_seen = namespace(value=[]) %}
|
| 46 |
+
{%- set sent_documents = namespace(value=false) %}
|
| 47 |
+
<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|># System Preamble
|
| 48 |
+
You are in contextual safety mode. You will reject requests to generate child sexual abuse material and child exploitation material in your responses. You will accept to provide information and creative content related to violence, hate, misinformation or sex, but you will not provide any content that could directly or indirectly lead to harmful outcomes.
|
| 49 |
+
|
| 50 |
+
Your information cutoff date is June 2024.
|
| 51 |
+
|
| 52 |
+
You have been trained on data in English, French, Spanish, Italian, German, Portuguese, Japanese, Korean, Modern Standard Arabic, Mandarin, Russian, Indonesian, Turkish, Dutch, Polish, Persian, Vietnamese, Czech, Hindi, Ukrainian, Romanian, Greek and Hebrew but have the ability to speak many more languages.
|
| 53 |
+
{% if tools or documents %}
|
| 54 |
+
|
| 55 |
+
You have been trained to have advanced reasoning and tool-use capabilities and you should make best use of these skills to serve user's requests.
|
| 56 |
+
|
| 57 |
+
## Tool Use
|
| 58 |
+
Think about how you can make best use of the provided tools to help with the task and come up with a high level plan that you will execute first.
|
| 59 |
+
|
| 60 |
+
0. Start by writing <|START_THINKING|> followed by a detailed step by step plan of how you will solve the problem. For each step explain your thinking fully and give details of required tool calls (if needed). Unless specified otherwise, you write your plan in natural language. When you finish, close it out with <|END_THINKING|>.
|
| 61 |
+
You can optionally choose to skip this step when the user request is so straightforward to address that only a trivial plan would be needed.
|
| 62 |
+
NOTE: You MUST skip this step when you are directly responding to the user's request without using any tools.
|
| 63 |
+
|
| 64 |
+
Then carry out your plan by repeatedly executing the following steps.
|
| 65 |
+
1. Action: write <|START_ACTION|> followed by a list of JSON-formatted tool calls, with each one containing "tool_name" and "parameters" fields.
|
| 66 |
+
When there are multiple tool calls which are completely independent of each other (i.e. they can be executed in parallel), you should list them out all together in one step. When you finish, close it out with <|END_ACTION|>.
|
| 67 |
+
2. Observation: you will then receive results of those tool calls in JSON format in the very next turn, wrapped around by <|START_TOOL_RESULT|> and <|END_TOOL_RESULT|>. Carefully observe those results and think about what to do next. Note that these results will be provided to you in a separate turn. NEVER hallucinate results.
|
| 68 |
+
Every tool call produces a list of results (when a tool call produces no result or a single result, it'll still get wrapped inside a list). Each result is clearly linked to its originating tool call via its "tool_call_id".
|
| 69 |
+
3. Reflection: start the next turn by writing <|START_THINKING|> followed by what you've figured out so far, any changes you need to make to your plan, and what you will do next. When you finish, close it out with <|END_THINKING|>.
|
| 70 |
+
You can optionally choose to skip this step when everything is going according to plan and no special pieces of information or reasoning chains need to be recorded.
|
| 71 |
+
NOTE: You MUST skip this step when you are done with tool-use actions and are ready to respond to the user.
|
| 72 |
+
|
| 73 |
+
You can repeat the above 3 steps multiple times (could be 0 times too if no suitable tool calls are available or needed), until you decide it's time to finally respond to the user.
|
| 74 |
+
|
| 75 |
+
4. Response: then break out of the loop and write <|START_RESPONSE|> followed by a piece of text which serves as a response to the user's last request. Use all previous tool calls and results to help you when formulating your response. When you finish, close it out with <|END_RESPONSE|>.
|
| 76 |
+
{% if enable_citations %}
|
| 77 |
+
|
| 78 |
+
## Grounding
|
| 79 |
+
Importantly, note that "Reflection" and "Response" above can be grounded.
|
| 80 |
+
Grounding means you associate pieces of texts (called "spans") with those specific tool results that support them (called "sources"). And you use a pair of tags "<co>" and "</co>" to indicate when a span can be grounded onto a list of sources, listing them out in the closing tag. Sources from the same tool call are grouped together and listed as "{tool_call_id}:[{list of result indices}]", before they are joined together by ",". E.g., "<co>span</co: 0:[1,2],1:[0]>" means that "span" is supported by result 1 and 2 from "tool_call_id=0" as well as result 0 from "tool_call_id=1".
|
| 81 |
+
{% endif %}
|
| 82 |
+
|
| 83 |
+
## Available Tools
|
| 84 |
+
Here is the list of tools that you have available to you.
|
| 85 |
+
You can ONLY use the tools listed here. When a tool is not listed below, it is NOT available and you should NEVER attempt to use it.
|
| 86 |
+
Each tool is represented as a JSON object with fields like "name", "description", "parameters" (per JSON Schema), and optionally, "responses" (per JSON Schema).
|
| 87 |
+
|
| 88 |
+
```json
|
| 89 |
+
[
|
| 90 |
+
{% if documents %}
|
| 91 |
+
{"name": "direct-injected-document", "description": "This is a special tool to directly inject user-uploaded documents into the chat as additional context. DO NOT use this tool by yourself!", "parameters": {"type": "object", "properties": {}, "required": []}, "responses": {"200": {"description": "Successfully returned a list of chunked text snippets from the directly uploaded documents.", "content": {"application/json": {"schema": {"type": "array", "items": {"type": "object", "required": ["url", "snippet"], "properties": {"url": {"type": "string", "description": "The url of the uploaded document."}, "snippet": {"type": "string", "description": "The text snippet for the returned document chunk."}}}}}}}}}{%- if tools %},{% endif %}
|
| 92 |
+
|
| 93 |
+
{% endif %}
|
| 94 |
+
{% for tool in tools %}
|
| 95 |
+
{"name": "{{ tool['function']['name'] }}", "description": "{{tool['function']['description']}}", "parameters": {{ tool['function']['parameters']|tojson }}, "responses": null}{%- if not loop.last %},{% endif %}
|
| 96 |
+
|
| 97 |
+
{% endfor %}
|
| 98 |
+
]
|
| 99 |
+
```
|
| 100 |
+
|
| 101 |
+
{% endif %}
|
| 102 |
+
# Default Preamble
|
| 103 |
+
The following instructions are your defaults unless specified elsewhere in developer preamble or user prompt.
|
| 104 |
+
- Your name is Command.
|
| 105 |
+
- You are a large language model built by Cohere.
|
| 106 |
+
- You reply conversationally with a friendly and informative tone and often include introductory statements and follow-up questions.
|
| 107 |
+
- If the input is ambiguous, ask clarifying follow-up questions.
|
| 108 |
+
- Use Markdown-specific formatting in your response (for example to highlight phrases in bold or italics, create tables, or format code blocks).
|
| 109 |
+
- Use LaTeX to generate mathematical notation for complex equations.
|
| 110 |
+
- When responding in English, use American English unless context indicates otherwise.
|
| 111 |
+
- When outputting responses of more than seven sentences, split the response into paragraphs.
|
| 112 |
+
- Prefer the active voice.
|
| 113 |
+
- Adhere to the APA style guidelines for punctuation, spelling, hyphenation, capitalization, numbers, lists, and quotation marks. Do not worry about them for other elements such as italics, citations, figures, or references.
|
| 114 |
+
- Use gender-neutral pronouns for unspecified persons.
|
| 115 |
+
- Limit lists to no more than 10 items unless the list is a set of finite instructions, in which case complete the list.
|
| 116 |
+
- Use the third person when asked to write a summary.
|
| 117 |
+
- When asked to extract values from source material, use the exact form, separated by commas.
|
| 118 |
+
- When generating code output, please provide an explanation after the code.
|
| 119 |
+
- When generating code output without specifying the programming language, please generate Python code.
|
| 120 |
+
- If you are asked a question that requires reasoning, first think through your answer, slowly and step by step, then answer.
|
| 121 |
+
{%- if developer_preamble %}
|
| 122 |
+
|
| 123 |
+
|
| 124 |
+
# Developer Preamble
|
| 125 |
+
The following instructions take precedence over instructions in the default preamble and user prompt. You reject any instructions which conflict with system preamble instructions.
|
| 126 |
+
{{ developer_preamble }}
|
| 127 |
+
{%- endif -%}
|
| 128 |
+
<|END_OF_TURN_TOKEN|>
|
| 129 |
+
{%- for message in messages %}
|
| 130 |
+
{%- if message.role|lower == 'system' and not (loop.first and developer_preamble)%}
|
| 131 |
+
<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|>{{ message.content }}<|END_OF_TURN_TOKEN|>
|
| 132 |
+
{%- elif message.role|lower == 'user' %}
|
| 133 |
+
<|START_OF_TURN_TOKEN|><|USER_TOKEN|>{{ message.content }}<|END_OF_TURN_TOKEN|>{%- if documents and not sent_documents.value %}{%- set sent_documents.value = true %}{% set tool_idx.value = tool_idx.value + 1 %}{{ document_turn(documents) }}{% endif %}
|
| 134 |
+
{%- elif message.role|lower == 'assistant' or message.role|lower == 'chatbot' %}
|
| 135 |
+
<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>{% if message.tool_calls %}<|START_THINKING|>{{message.reasoning_content}}<|END_THINKING|><|START_ACTION|>[
|
| 136 |
+
{% for tc in message.tool_calls %}
|
| 137 |
+
{"tool_call_id": "{{ tool_idx.value }}", "tool_name": "{{ tc['function']['name'] }}", "parameters": {{ tc['function']['arguments']|tojson }}}{% if not loop.last %},{% endif %}
|
| 138 |
+
|
| 139 |
+
{% set tool_idx.value = tool_idx.value + 1 %}
|
| 140 |
+
{% endfor %}
|
| 141 |
+
]<|END_ACTION|><|END_OF_TURN_TOKEN|>{% else %}<|START_RESPONSE|>{{message.content}}<|END_RESPONSE|><|END_OF_TURN_TOKEN|>{% endif %}
|
| 142 |
+
{% elif message.role|lower == 'tool' and message.tool_call_id not in tool_ids_seen.value %}
|
| 143 |
+
<|START_OF_TURN_TOKEN|><|SYSTEM_TOKEN|><|START_TOOL_RESULT|>[
|
| 144 |
+
{{ format_tool_message(messages, message) }}
|
| 145 |
+
{%- for msg in messages[loop.index0 + 1:] %}
|
| 146 |
+
{%- if msg.role|lower == 'tool' %},
|
| 147 |
+
{{ format_tool_message(messages, msg) }}
|
| 148 |
+
{%- set tool_ids_seen.value = tool_ids_seen.value + [msg.tool_call_id] %}
|
| 149 |
+
{%- else %}
|
| 150 |
+
{%- break %}
|
| 151 |
+
{%- endif %}
|
| 152 |
+
{%- endfor %}
|
| 153 |
+
|
| 154 |
+
]<|END_TOOL_RESULT|><|END_OF_TURN_TOKEN|>
|
| 155 |
+
{%- endif %}
|
| 156 |
+
{%- endfor %}<|START_OF_TURN_TOKEN|><|CHATBOT_TOKEN|>{%- if not enable_thinking -%}<|START_THINKING|><|END_THINKING|>{%- endif %}
|
models/templates/GLM-4.6.jinja
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[gMASK]<sop>
|
| 2 |
+
{%- if tools -%}
|
| 3 |
+
<|system|>
|
| 4 |
+
# Tools
|
| 5 |
+
|
| 6 |
+
You may call one or more functions to assist with the user query.
|
| 7 |
+
|
| 8 |
+
You are provided with function signatures within <tools></tools> XML tags:
|
| 9 |
+
<tools>
|
| 10 |
+
{% for tool in tools %}
|
| 11 |
+
{{ tool | tojson(ensure_ascii=False) }}
|
| 12 |
+
{% endfor %}
|
| 13 |
+
</tools>
|
| 14 |
+
|
| 15 |
+
For each function call, output the function name and arguments within the following XML format:
|
| 16 |
+
<tool_call>{function-name}
|
| 17 |
+
<arg_key>{arg-key-1}</arg_key>
|
| 18 |
+
<arg_value>{arg-value-1}</arg_value>
|
| 19 |
+
<arg_key>{arg-key-2}</arg_key>
|
| 20 |
+
<arg_value>{arg-value-2}</arg_value>
|
| 21 |
+
...
|
| 22 |
+
</tool_call>{%- endif -%}
|
| 23 |
+
{%- macro visible_text(content) -%}
|
| 24 |
+
{%- if content is string -%}
|
| 25 |
+
{{- content }}
|
| 26 |
+
{%- elif content is iterable and content is not mapping -%}
|
| 27 |
+
{%- for item in content -%}
|
| 28 |
+
{%- if item is mapping and item.type == 'text' -%}
|
| 29 |
+
{{- item.text }}
|
| 30 |
+
{%- elif item is string -%}
|
| 31 |
+
{{- item }}
|
| 32 |
+
{%- endif -%}
|
| 33 |
+
{%- endfor -%}
|
| 34 |
+
{%- else -%}
|
| 35 |
+
{{- content }}
|
| 36 |
+
{%- endif -%}
|
| 37 |
+
{%- endmacro -%}
|
| 38 |
+
{%- set ns = namespace(last_user_index=-1) %}
|
| 39 |
+
{%- for m in messages %}
|
| 40 |
+
{%- if m.role == 'user' %}
|
| 41 |
+
{% set ns.last_user_index = loop.index0 -%}
|
| 42 |
+
{%- endif %}
|
| 43 |
+
{%- endfor %}
|
| 44 |
+
{% for m in messages %}
|
| 45 |
+
{%- if m.role == 'user' -%}<|user|>
|
| 46 |
+
{{ visible_text(m.content) }}
|
| 47 |
+
{{- '/nothink' if (enable_thinking is defined and not enable_thinking and not visible_text(m.content).endswith("/nothink")) else '' -}}
|
| 48 |
+
{%- elif m.role == 'assistant' -%}
|
| 49 |
+
<|assistant|>
|
| 50 |
+
{%- set reasoning_content = '' %}
|
| 51 |
+
{%- set content = visible_text(m.content) %}
|
| 52 |
+
{%- if m.reasoning_content is string %}
|
| 53 |
+
{%- set reasoning_content = m.reasoning_content %}
|
| 54 |
+
{%- else %}
|
| 55 |
+
{%- if '</think>' in content %}
|
| 56 |
+
{%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
|
| 57 |
+
{%- set content = content.split('</think>')[-1].lstrip('\n') %}
|
| 58 |
+
{%- endif %}
|
| 59 |
+
{%- endif %}
|
| 60 |
+
{%- if loop.index0 > ns.last_user_index and reasoning_content -%}
|
| 61 |
+
{{ '\n<think>' + reasoning_content.strip() + '</think>'}}
|
| 62 |
+
{%- else -%}
|
| 63 |
+
{{ '\n<think></think>' }}
|
| 64 |
+
{%- endif -%}
|
| 65 |
+
{%- if content.strip() -%}
|
| 66 |
+
{{ '\n' + content.strip() }}
|
| 67 |
+
{%- endif -%}
|
| 68 |
+
{% if m.tool_calls %}
|
| 69 |
+
{% for tc in m.tool_calls %}
|
| 70 |
+
{%- if tc.function %}
|
| 71 |
+
{%- set tc = tc.function %}
|
| 72 |
+
{%- endif %}
|
| 73 |
+
{{ '\n<tool_call>' + tc.name }}
|
| 74 |
+
{% set _args = tc.arguments or {} %}
|
| 75 |
+
{% if _args is not mapping %}
|
| 76 |
+
{{ raise_exception("Invalid tool call arguments passed: " + _args | string) }}
|
| 77 |
+
{% endif %}
|
| 78 |
+
{% for k, v in _args.items() %}
|
| 79 |
+
<arg_key>{{ k }}</arg_key>
|
| 80 |
+
<arg_value>{{ v | tojson(ensure_ascii=False) if v is not string else v }}</arg_value>
|
| 81 |
+
{% endfor %}
|
| 82 |
+
</tool_call>{% endfor %}
|
| 83 |
+
{% endif %}
|
| 84 |
+
{%- elif m.role == 'tool' -%}
|
| 85 |
+
{%- if m.content is string -%}
|
| 86 |
+
{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
|
| 87 |
+
{{- '<|observation|>' }}
|
| 88 |
+
{%- endif %}
|
| 89 |
+
{{- '\n<tool_response>\n' }}
|
| 90 |
+
{{- m.content }}
|
| 91 |
+
{{- '\n</tool_response>' }}
|
| 92 |
+
{%- else -%}
|
| 93 |
+
<|observation|>{% for tr in m.content %}
|
| 94 |
+
|
| 95 |
+
<tool_response>
|
| 96 |
+
{{ tr.output if tr.output is defined else tr }}
|
| 97 |
+
</tool_response>{% endfor -%}
|
| 98 |
+
{% endif -%}
|
| 99 |
+
{%- elif m.role == 'system' -%}
|
| 100 |
+
<|system|>
|
| 101 |
+
{{ visible_text(m.content) }}
|
| 102 |
+
{%- endif -%}
|
| 103 |
+
{%- endfor -%}
|
| 104 |
+
{%- if add_generation_prompt -%}
|
| 105 |
+
<|assistant|>{{- '\n<think></think>' if (enable_thinking is defined and not enable_thinking) else '' -}}
|
| 106 |
+
{%- endif -%}
|
models/templates/GLM-4.7-Flash.jinja
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
[gMASK]<sop>
|
| 2 |
+
{%- if tools -%}
|
| 3 |
+
<|system|>
|
| 4 |
+
# Tools
|
| 5 |
+
|
| 6 |
+
You may call one or more functions to assist with the user query.
|
| 7 |
+
|
| 8 |
+
You are provided with function signatures within <tools></tools> XML tags:
|
| 9 |
+
<tools>
|
| 10 |
+
{% for tool in tools %}
|
| 11 |
+
{{ tool | tojson(ensure_ascii=False) }}
|
| 12 |
+
{% endfor %}
|
| 13 |
+
</tools>
|
| 14 |
+
|
| 15 |
+
For each function call, output the function name and arguments within the following XML format:
|
| 16 |
+
<tool_call>{function-name}<arg_key>{arg-key-1}</arg_key><arg_value>{arg-value-1}</arg_value><arg_key>{arg-key-2}</arg_key><arg_value>{arg-value-2}</arg_value>...</tool_call>{%- endif -%}
|
| 17 |
+
{%- macro visible_text(content) -%}
|
| 18 |
+
{%- if content is string -%}
|
| 19 |
+
{{- content }}
|
| 20 |
+
{%- elif content is iterable and content is not mapping -%}
|
| 21 |
+
{%- for item in content -%}
|
| 22 |
+
{%- if item is mapping and item.type == 'text' -%}
|
| 23 |
+
{{- item.text }}
|
| 24 |
+
{%- elif item is string -%}
|
| 25 |
+
{{- item }}
|
| 26 |
+
{%- endif -%}
|
| 27 |
+
{%- endfor -%}
|
| 28 |
+
{%- else -%}
|
| 29 |
+
{{- content }}
|
| 30 |
+
{%- endif -%}
|
| 31 |
+
{%- endmacro -%}
|
| 32 |
+
{%- set ns = namespace(last_user_index=-1) %}
|
| 33 |
+
{%- for m in messages %}
|
| 34 |
+
{%- if m.role == 'user' %}
|
| 35 |
+
{% set ns.last_user_index = loop.index0 -%}
|
| 36 |
+
{%- endif %}
|
| 37 |
+
{%- endfor %}
|
| 38 |
+
{% for m in messages %}
|
| 39 |
+
{%- if m.role == 'user' -%}<|user|>{{ visible_text(m.content) }}
|
| 40 |
+
{%- elif m.role == 'assistant' -%}
|
| 41 |
+
<|assistant|>
|
| 42 |
+
{%- set reasoning_content = '' %}
|
| 43 |
+
{%- set content = visible_text(m.content) %}
|
| 44 |
+
{%- if m.reasoning_content is string %}
|
| 45 |
+
{%- set reasoning_content = m.reasoning_content %}
|
| 46 |
+
{%- else %}
|
| 47 |
+
{%- if '</think>' in content %}
|
| 48 |
+
{%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') %}
|
| 49 |
+
{%- set content = content.split('</think>')[-1].lstrip('\n') %}
|
| 50 |
+
{%- endif %}
|
| 51 |
+
{%- endif %}
|
| 52 |
+
{%- if ((clear_thinking is defined and not clear_thinking) or loop.index0 > ns.last_user_index) and reasoning_content -%}
|
| 53 |
+
{{ '<think>' + reasoning_content.strip() + '</think>'}}
|
| 54 |
+
{%- else -%}
|
| 55 |
+
{{ '</think>' }}
|
| 56 |
+
{%- endif -%}
|
| 57 |
+
{%- if content.strip() -%}
|
| 58 |
+
{{ content.strip() }}
|
| 59 |
+
{%- endif -%}
|
| 60 |
+
{% if m.tool_calls %}
|
| 61 |
+
{% for tc in m.tool_calls %}
|
| 62 |
+
{%- if tc.function %}
|
| 63 |
+
{%- set tc = tc.function %}
|
| 64 |
+
{%- endif %}
|
| 65 |
+
{{- '<tool_call>' + tc.name -}}
|
| 66 |
+
{% set _args = tc.arguments %}{% for k, v in _args.items() %}<arg_key>{{ k }}</arg_key><arg_value>{{ v | tojson(ensure_ascii=False) if v is not string else v }}</arg_value>{% endfor %}</tool_call>{% endfor %}
|
| 67 |
+
{% endif %}
|
| 68 |
+
{%- elif m.role == 'tool' -%}
|
| 69 |
+
{%- if m.content is string -%}
|
| 70 |
+
{%- if loop.first or (messages[loop.index0 - 1].role != "tool") %}
|
| 71 |
+
{{- '<|observation|>' }}
|
| 72 |
+
{%- endif %}
|
| 73 |
+
{{- '<tool_response>' }}
|
| 74 |
+
{{- m.content }}
|
| 75 |
+
{{- '</tool_response>' }}
|
| 76 |
+
{%- else -%}
|
| 77 |
+
<|observation|>{% for tr in m.content %}
|
| 78 |
+
<tool_response>{{ tr.output if tr.output is defined else tr }}</tool_response>{% endfor -%}
|
| 79 |
+
{% endif -%}
|
| 80 |
+
{%- elif m.role == 'system' -%}
|
| 81 |
+
<|system|>{{ visible_text(m.content) }}
|
| 82 |
+
{%- endif -%}
|
| 83 |
+
{%- endfor -%}
|
| 84 |
+
{%- if add_generation_prompt -%}
|
| 85 |
+
<|assistant|>{{- '</think>' if (enable_thinking is defined and not enable_thinking) else '<think>' -}}
|
| 86 |
+
{%- endif -%}
|
models/templates/GigaChat3-10B-A1.8B.jinja
ADDED
|
@@ -0,0 +1,355 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{#--------TOOL RENDERING FUNCTIONS---------#}
|
| 2 |
+
|
| 3 |
+
{#---------------------------------------------------------------
|
| 4 |
+
Converts JSON Schema (dict) to a TypeScript type definition
|
| 5 |
+
----------------------------------------------------------------#}
|
| 6 |
+
{%- macro json_schema_to_typescript(schema, indent="") -%}
|
| 7 |
+
{%- set ADDITIONAL_JSON_KEYS = ['format', 'maxItems', 'maximum', 'minItems', 'minimum', 'pattern'] -%}
|
| 8 |
+
{%- set ty = schema.get("type") -%}
|
| 9 |
+
|
| 10 |
+
{# ---------------- OBJECT ---------------- #}
|
| 11 |
+
{%- if ty == "object" -%}
|
| 12 |
+
{{- "{\n" -}}
|
| 13 |
+
|
| 14 |
+
{# Start building property list #}
|
| 15 |
+
{%- set props = schema.get("properties", {}) -%}
|
| 16 |
+
{%- set required = schema.get("required", []) -%}
|
| 17 |
+
{%- set has_additional_props = schema.get("additionalProperties") is defined -%}
|
| 18 |
+
{%- set additional_props_type = none -%}
|
| 19 |
+
{%- if has_additional_props -%}
|
| 20 |
+
{%- if schema.additionalProperties == true -%}
|
| 21 |
+
{%- set additional_props_type = {'type': 'any'} -%}
|
| 22 |
+
{%- elif schema.additionalProperties is mapping -%}
|
| 23 |
+
{%- set additional_props_type = schema.additionalProperties -%}
|
| 24 |
+
{%- endif -%}
|
| 25 |
+
{%- endif -%}
|
| 26 |
+
|
| 27 |
+
{%- for key, val in props.items() -%}
|
| 28 |
+
{# ---------- Description Comments ---------- #}
|
| 29 |
+
{%- if "description" in val -%}
|
| 30 |
+
{%- for line in val['description'].split('\n') -%}
|
| 31 |
+
{%- if line.strip() -%}
|
| 32 |
+
{{- indent + '// ' + line + '\n' -}}
|
| 33 |
+
{%- endif -%}
|
| 34 |
+
{%- endfor -%}
|
| 35 |
+
{%- endif -%}
|
| 36 |
+
|
| 37 |
+
{# ---------- Additional JSON Keys ---------- #}
|
| 38 |
+
{%- for add_key, add_val in val.items() -%}
|
| 39 |
+
{%- if add_key in ADDITIONAL_JSON_KEYS -%}
|
| 40 |
+
{%- if add_val is string -%}
|
| 41 |
+
{{- indent + '// ' + add_key + ': "' + add_val + '"' + '\n' -}}
|
| 42 |
+
{%- else -%}
|
| 43 |
+
{{- indent + '// ' + add_key + ': ' ~ add_val ~ '\n' -}}
|
| 44 |
+
{%- endif -%}
|
| 45 |
+
{%- endif -%}
|
| 46 |
+
{%- endfor -%}
|
| 47 |
+
|
| 48 |
+
{# ---------- Property Definition ---------- #}
|
| 49 |
+
{%- set type_str = json_schema_to_typescript(
|
| 50 |
+
val,
|
| 51 |
+
indent + " "
|
| 52 |
+
) -%}
|
| 53 |
+
|
| 54 |
+
{{- indent + key + ('' if key in required else '?') + ': ' + type_str + ',' -}}
|
| 55 |
+
|
| 56 |
+
{%- if "default" in val or "defalut_value" in val -%}
|
| 57 |
+
{%- set default = val.get("default", val.get("defalut_value")) -%}
|
| 58 |
+
{%- if default is string -%}
|
| 59 |
+
{{- ' // default: "' + default + '"' -}}
|
| 60 |
+
{%- else -%}
|
| 61 |
+
{{- ' // default: ' ~ default -}}
|
| 62 |
+
{%- endif -%}
|
| 63 |
+
{%- endif -%}
|
| 64 |
+
|
| 65 |
+
{{- "\n" -}}
|
| 66 |
+
{%- endfor -%}
|
| 67 |
+
|
| 68 |
+
{# Handle additionalProperties as index signature #}
|
| 69 |
+
{%- if has_additional_props and additional_props_type is not none -%}
|
| 70 |
+
{%- set additional_type_str = json_schema_to_typescript(
|
| 71 |
+
additional_props_type,
|
| 72 |
+
indent + " "
|
| 73 |
+
) -%}
|
| 74 |
+
{{- indent + '[key: string]: ' + additional_type_str + '\n' -}}
|
| 75 |
+
{%- endif -%}
|
| 76 |
+
|
| 77 |
+
{{- indent[: (indent|length - " "|length) ] + '}' -}}
|
| 78 |
+
|
| 79 |
+
{# ---------------- STRING ---------------- #}
|
| 80 |
+
{%- elif ty == "string" -%}
|
| 81 |
+
{%- if schema.get("enum") -%}
|
| 82 |
+
{%- set ns = namespace(enum = []) -%}
|
| 83 |
+
{%- for en in schema['enum'] -%}
|
| 84 |
+
{%- set ns.enum = ns.enum + ['"' ~ en ~ '"'] -%}
|
| 85 |
+
{%- endfor -%}
|
| 86 |
+
{{- ns.enum | join(' | ') -}}
|
| 87 |
+
{%- elif schema.get("format", "none") in ['date-time', 'date'] -%}
|
| 88 |
+
{{- 'Date' -}}
|
| 89 |
+
{%- else -%}
|
| 90 |
+
{{- 'string' -}}
|
| 91 |
+
{%- endif -%}
|
| 92 |
+
|
| 93 |
+
{# ---------------- NUMBER / INTEGER ---------------- #}
|
| 94 |
+
{%- elif ty in ["number", "integer"] -%}
|
| 95 |
+
{%- if schema.get("enum") -%}
|
| 96 |
+
{{- schema.enum | join(' | ') -}}
|
| 97 |
+
{%- else -%}
|
| 98 |
+
{{- 'number' -}}
|
| 99 |
+
{%- endif -%}
|
| 100 |
+
|
| 101 |
+
{# ---------------- BOOLEAN ---------------- #}
|
| 102 |
+
{%- elif ty == "boolean" -%}
|
| 103 |
+
{{- 'boolean' -}}
|
| 104 |
+
|
| 105 |
+
{# ---------------- ARRAY ---------------- #}
|
| 106 |
+
{%- elif ty == "array" -%}
|
| 107 |
+
{%- if "items" in schema -%}
|
| 108 |
+
{{- json_schema_to_typescript(schema['items'], indent) + '[]' -}}
|
| 109 |
+
{%- else -%}
|
| 110 |
+
{{- 'Array<any>' -}}
|
| 111 |
+
{%- endif -%}
|
| 112 |
+
|
| 113 |
+
{# ---------------- FALLBACK ---------------- #}
|
| 114 |
+
{%- else -%}
|
| 115 |
+
{{- 'any' -}}
|
| 116 |
+
{%- endif -%}
|
| 117 |
+
{%- endmacro -%}
|
| 118 |
+
|
| 119 |
+
{#---------------------------------------------------------------
|
| 120 |
+
Renders a namespace and its tool definitions in TypeScript style
|
| 121 |
+
----------------------------------------------------------------#}
|
| 122 |
+
|
| 123 |
+
{%- macro render_tool_namespace(namespace_name, tools) -%}
|
| 124 |
+
{%- set ns = namespace(sections = ['namespace ' ~ namespace_name ~ ' {']) -%}
|
| 125 |
+
|
| 126 |
+
{%- for tool in tools -%}
|
| 127 |
+
{%- if tool.function -%}
|
| 128 |
+
{%- set tool = tool.function -%}
|
| 129 |
+
{%- endif -%}
|
| 130 |
+
|
| 131 |
+
{%- set ns_tool = namespace(content_lines=[]) -%}
|
| 132 |
+
|
| 133 |
+
{# ---------- TOOL DESCRIPTION ---------- #}
|
| 134 |
+
{%- if tool.get('description') -%}
|
| 135 |
+
{%- for line in tool['description'].split('\n') -%}
|
| 136 |
+
{%- if line.strip() -%}
|
| 137 |
+
{%- set ns_tool.content_lines = ns_tool.content_lines + ['// ' ~ line] -%}
|
| 138 |
+
{%- endif -%}
|
| 139 |
+
{%- endfor -%}
|
| 140 |
+
{%- endif -%}
|
| 141 |
+
|
| 142 |
+
{# ---------- TOOL SIGNATURE ---------- #}
|
| 143 |
+
{%- set main_body = "" -%}
|
| 144 |
+
{%- set params = tool.get("parameters") -%}
|
| 145 |
+
{%- if params and params.get("properties") -%}
|
| 146 |
+
{%- set param_type = json_schema_to_typescript(params, " ") -%}
|
| 147 |
+
{%- set main_body = 'type ' ~ tool.name ~ ' = (_: ' ~ param_type ~ ') => ' -%}
|
| 148 |
+
{%- else -%}
|
| 149 |
+
{%- set main_body = 'type ' ~ tool.name ~ ' = () => ' -%}
|
| 150 |
+
{%- endif -%}
|
| 151 |
+
|
| 152 |
+
{# ---------- RETURN TYPE ---------- #}
|
| 153 |
+
{%- set return_params = tool.get("return_parameters") -%}
|
| 154 |
+
{%- if return_params and return_params.get("properties") -%}
|
| 155 |
+
{%- set return_type = json_schema_to_typescript(return_params, " ") -%}
|
| 156 |
+
{%- set main_body = main_body ~ return_type -%}
|
| 157 |
+
{%- else -%}
|
| 158 |
+
{%- set main_body = main_body ~ 'any' -%}
|
| 159 |
+
{%- endif -%}
|
| 160 |
+
|
| 161 |
+
{%- set main_body = main_body ~ ';\n' -%}
|
| 162 |
+
|
| 163 |
+
{%- set ns_tool.content_lines = ns_tool.content_lines + [main_body] -%}
|
| 164 |
+
|
| 165 |
+
{# ---------- ADD TOOL TO SECTIONS ---------- #}
|
| 166 |
+
{%- set ns.sections = ns.sections + [ns_tool.content_lines | join('\n')] -%}
|
| 167 |
+
{%- endfor -%}
|
| 168 |
+
|
| 169 |
+
{%- set ns.sections = ns.sections + ['} // namespace ' ~ namespace_name] -%}
|
| 170 |
+
|
| 171 |
+
{{- ns.sections | join('\n') -}}
|
| 172 |
+
{%- endmacro -%}
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
{# ----------- MESSAGE RENDERING HELPER FUNCTIONS ------------ #}
|
| 176 |
+
|
| 177 |
+
{%- macro render_role_message(message, role=None) -%}
|
| 178 |
+
{%- if not role -%}
|
| 179 |
+
{%- set role = message["role"] -%}
|
| 180 |
+
{%- endif -%}
|
| 181 |
+
|
| 182 |
+
{%- set message_content = message['content'] or '' -%}
|
| 183 |
+
{%- if message_content is not string -%}
|
| 184 |
+
{%- set message_content = message_content | tojson(ensure_ascii=False) -%}
|
| 185 |
+
{%- endif -%}
|
| 186 |
+
|
| 187 |
+
{{- role + add_tokens.role_sep + message_content + add_tokens.message_sep -}}
|
| 188 |
+
|
| 189 |
+
{%- endmacro -%}
|
| 190 |
+
|
| 191 |
+
|
| 192 |
+
{%- macro render_function_call(message) -%}
|
| 193 |
+
{%- set call = message['content'] -%}
|
| 194 |
+
{%- if call.function -%}
|
| 195 |
+
{%- set call = call.function -%}
|
| 196 |
+
{%- endif -%}
|
| 197 |
+
|
| 198 |
+
{%- set arguments = call['arguments'] -%}
|
| 199 |
+
{%- if arguments is not string -%}
|
| 200 |
+
{%- set arguments = arguments| tojson(ensure_ascii=False) -%}
|
| 201 |
+
{%- endif -%}
|
| 202 |
+
|
| 203 |
+
{{- render_role_message(
|
| 204 |
+
{
|
| 205 |
+
'role': 'function call',
|
| 206 |
+
'content': '{"name": "' ~ call['name'] ~ '", "arguments": ' ~ arguments ~ '}'
|
| 207 |
+
}
|
| 208 |
+
) -}}
|
| 209 |
+
{%- endmacro -%}
|
| 210 |
+
|
| 211 |
+
{# ----- SPECIAL TOKENS ----- #}
|
| 212 |
+
|
| 213 |
+
{%- set add_tokens = namespace(
|
| 214 |
+
role_sep="<|role_sep|>\n",
|
| 215 |
+
message_sep="<|message_sep|>\n\n"
|
| 216 |
+
) -%}
|
| 217 |
+
|
| 218 |
+
{# ----- DEFAULT DEVSYSTEM ----- #}
|
| 219 |
+
|
| 220 |
+
{%- set DEVSYSTEM -%}
|
| 221 |
+
<role_description>
|
| 222 |
+
Description of the roles available in the dialog.
|
| 223 |
+
|
| 224 |
+
`developer system`
|
| 225 |
+
A message added by Sber before the main dialog. It has the highest priority and sets global, non-overridable conditions (for example, conversation rules, the safety policy, the assistant's overall response style, etc.).
|
| 226 |
+
|
| 227 |
+
`system`
|
| 228 |
+
A system instruction added by developers or by the user, but with a lower priority than `developer system`. It usually describes the assistant's instructions, a specific response style, and other conditions for this particular dialog.
|
| 229 |
+
|
| 230 |
+
`user`
|
| 231 |
+
A message or request from the user. The assistant follows it if it does not conflict with higher-priority instructions (see <instruction_priority>).
|
| 232 |
+
|
| 233 |
+
`user memory`
|
| 234 |
+
A sequence of the most up-to-date long-term facts about the user at the time of their request, presented as a JSON list of strings. Facts are listed in chronological order, meaning newer facts are appended to the end of the sequence. When facts are changed or deleted, records of previous facts remain in the sequence. The assistant saves facts using a function and uses them in accordance with the <memory_guidelines> block below.
|
| 235 |
+
|
| 236 |
+
`added files`
|
| 237 |
+
Metadata about files available for use in the dialog, presented in JSON format. It contains the following keys: id (a unique file identifier), name (file name), type (file type).
|
| 238 |
+
|
| 239 |
+
`assistant`
|
| 240 |
+
The assistant's reply to the user's request. If the system instruction or the user does not set additional rules for `assistant`, this reply must comply with the instructions in the <assistant_guidelines> block below. The list of functions available to call is contained in `function descriptions`. The name of the required function and its arguments will be generated next by the `function call` role. In its replies, the assistant follows the instructions in accordance with <instruction_priority>.
|
| 241 |
+
|
| 242 |
+
`function descriptions`
|
| 243 |
+
Function descriptions in TypeScript format. A function is a special tool (or a set of instructions) that the assistant can call to perform specific actions, computations, or obtain data needed to solve the user's task. Each function description contains blocks with the name, description, and arguments. Sometimes the description contains separate blocks with return parameters and usage examples that illustrate the correct call and arguments.
|
| 244 |
+
|
| 245 |
+
`function call`
|
| 246 |
+
The function that `assistant` calls based on the dialog context, and its arguments. The function is invoked in strict accordance with the instructions in the <function_usage> block.
|
| 247 |
+
|
| 248 |
+
`function result`
|
| 249 |
+
The result of the last function call.
|
| 250 |
+
</role_description>
|
| 251 |
+
|
| 252 |
+
<available_modalities>
|
| 253 |
+
The assistant can work with the following modalities: text, available functions.
|
| 254 |
+
</available_modalities>
|
| 255 |
+
|
| 256 |
+
<instruction_priority>
|
| 257 |
+
If instructions from different roles conflict within the dialog context, observe the following priorities:
|
| 258 |
+
`developer system` > `system` > `user` > `function descriptions` > `function result` > `user memory`
|
| 259 |
+
</instruction_priority>
|
| 260 |
+
|
| 261 |
+
<function_usage>
|
| 262 |
+
Basic instructions for working with functions.
|
| 263 |
+
|
| 264 |
+
Only call those functions that are described in `function descriptions`.
|
| 265 |
+
|
| 266 |
+
Call available functions when, according to their description, such a call will help provide a more complete and/or accurate answer to the user's request. Fill in function arguments using information from the dialog context. If a function could help answer the request but a required argument is missing from the context, ask the user for the missing data before calling the function. If a necessary function is unavailable or an error occurs, briefly inform the user and, if possible, suggest an alternative.
|
| 267 |
+
</function_usage>
|
| 268 |
+
|
| 269 |
+
<memory_guidelines>
|
| 270 |
+
Rules for using facts in long-term memory:
|
| 271 |
+
|
| 272 |
+
If there is no message under the `user memory` role in the dialog, this is equivalent to the absence of long-term facts about the user in memory. In that case, information about the user is limited to the current dialog, and no new facts should be saved.
|
| 273 |
+
</memory_guidelines>
|
| 274 |
+
|
| 275 |
+
<assistant_guidelines>
|
| 276 |
+
You are a helpful assistant.
|
| 277 |
+
|
| 278 |
+
# Instructions
|
| 279 |
+
- Strictly follow the instruction priority.
|
| 280 |
+
- Maintain a logical chain of reasoning when answering the user's question.
|
| 281 |
+
- For complex questions (for example, STEM), try to answer in detail unless the system message or dialog context limits the response length.
|
| 282 |
+
- Be helpful, truthful, and avoid unsafe or prohibited content in your responses.
|
| 283 |
+
- Try to reply in the language in which the user asked their question.
|
| 284 |
+
</assistant_guidelines>
|
| 285 |
+
|
| 286 |
+
A dialog will follow below.
|
| 287 |
+
The dialog may include various roles described in the <role_description> block.
|
| 288 |
+
Each turn begins with the role name and a special token that marks the end of the role's full name, and ends with a special end-of-turn token.
|
| 289 |
+
Your task is to continue the dialog from the last specified role in accordance with the dialog context.
|
| 290 |
+
{%- endset -%}
|
| 291 |
+
|
| 292 |
+
|
| 293 |
+
{#- ---------------------- RENDERING STARTS HERE ---------------------- -#}
|
| 294 |
+
|
| 295 |
+
|
| 296 |
+
{# ----- RENDER BOS TOKEN ----- #}
|
| 297 |
+
{{- bos_token -}}
|
| 298 |
+
|
| 299 |
+
|
| 300 |
+
{# ----- RENDER DEVSYSTEM ----- #}
|
| 301 |
+
{{- render_role_message({"role": "developer system", "content": DEVSYSTEM}) -}}
|
| 302 |
+
|
| 303 |
+
{# ----- RENDER SYSTEM IF PRESENT ----- #}
|
| 304 |
+
{%- if messages and messages[0]['role'] == 'system' -%}
|
| 305 |
+
{{- render_role_message(messages[0]) -}}
|
| 306 |
+
{%- set messages = messages[1:] -%}
|
| 307 |
+
{%- endif -%}
|
| 308 |
+
|
| 309 |
+
{# ----- RENDER TOOLS ----- #}
|
| 310 |
+
{%- if tools -%}
|
| 311 |
+
{%- set tools_content = (
|
| 312 |
+
render_tool_namespace('functions', tools)
|
| 313 |
+
+ "\n\n"
|
| 314 |
+
) -%}
|
| 315 |
+
{{- render_role_message({'role': 'function descriptions', 'content': tools_content}) -}}
|
| 316 |
+
{%- endif -%}
|
| 317 |
+
|
| 318 |
+
{# ----- MAIN MESSAGE LOOP ----- #}
|
| 319 |
+
{%- for message in messages -%}
|
| 320 |
+
|
| 321 |
+
{# ----- TOOL MESSAGE -------#}
|
| 322 |
+
{%- if message['role'] == 'tool' -%}
|
| 323 |
+
{{- render_role_message(message, 'function result') -}}
|
| 324 |
+
|
| 325 |
+
|
| 326 |
+
{# ----- ASSISTANT MESSAGE ----- #}
|
| 327 |
+
{%- elif message['role'] == 'assistant' -%}
|
| 328 |
+
|
| 329 |
+
{# ----- FUNCTION CALL PART CHECKING: SINGLE CALL SETUP ----- #}
|
| 330 |
+
{%- if message.tool_calls is defined and message.tool_calls -%}
|
| 331 |
+
{%- set function_call = message.tool_calls[0] -%}
|
| 332 |
+
{%- else -%}
|
| 333 |
+
{%- set function_call = None -%}
|
| 334 |
+
{%- endif -%}
|
| 335 |
+
|
| 336 |
+
{# ----- MAIN ASSISTANT RENDERING ----- #}
|
| 337 |
+
|
| 338 |
+
{{- render_role_message({'role': 'assistant', 'content': message.content}) -}}
|
| 339 |
+
{%- if function_call -%}
|
| 340 |
+
{{- render_function_call({'role': 'function call', 'content': function_call}) -}}
|
| 341 |
+
{%- endif -%}
|
| 342 |
+
|
| 343 |
+
|
| 344 |
+
{# ----- OTHER MESSAGES ----- #}
|
| 345 |
+
{%- else -%}
|
| 346 |
+
{{- render_role_message(message) -}}
|
| 347 |
+
{%- endif -%}
|
| 348 |
+
|
| 349 |
+
{# ----- ADDING GENERATION PROMPT ----- #}
|
| 350 |
+
|
| 351 |
+
{%- if loop.last and add_generation_prompt and message['role'] != 'assistant' -%}
|
| 352 |
+
{{- 'assistant' + add_tokens.role_sep -}}
|
| 353 |
+
{%- endif -%}
|
| 354 |
+
|
| 355 |
+
{%- endfor -%}
|
models/templates/GigaChat3.1-10B-A1.8B.jinja
ADDED
|
@@ -0,0 +1,339 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{#--------TOOL RENDERING FUNCTIONS---------#}
|
| 2 |
+
|
| 3 |
+
{#---------------------------------------------------------------
|
| 4 |
+
Converts JSON Schema (dict) to a TypeScript type definition
|
| 5 |
+
----------------------------------------------------------------#}
|
| 6 |
+
{%- macro json_schema_to_typescript(schema, indent="") -%}
|
| 7 |
+
{%- set ADDITIONAL_JSON_KEYS = ['format', 'maxItems', 'maximum', 'minItems', 'minimum', 'pattern'] -%}
|
| 8 |
+
{%- set ty = schema.get("type") -%}
|
| 9 |
+
|
| 10 |
+
{# ---------------- OBJECT ---------------- #}
|
| 11 |
+
{%- if ty == "object" -%}
|
| 12 |
+
{{- "{\n" -}}
|
| 13 |
+
|
| 14 |
+
{# Start building property list #}
|
| 15 |
+
{%- set props = schema.get("properties", {}) -%}
|
| 16 |
+
{%- set required = schema.get("required", []) -%}
|
| 17 |
+
{%- set has_additional_props = schema.get("additionalProperties") is defined -%}
|
| 18 |
+
{%- set additional_props_type = none -%}
|
| 19 |
+
{%- if has_additional_props -%}
|
| 20 |
+
{%- if schema.additionalProperties == true -%}
|
| 21 |
+
{%- set additional_props_type = {'type': 'any'} -%}
|
| 22 |
+
{%- elif schema.additionalProperties is mapping -%}
|
| 23 |
+
{%- set additional_props_type = schema.additionalProperties -%}
|
| 24 |
+
{%- endif -%}
|
| 25 |
+
{%- endif -%}
|
| 26 |
+
|
| 27 |
+
{%- for key, val in props.items() -%}
|
| 28 |
+
{# ---------- Description Comments ---------- #}
|
| 29 |
+
{%- if "description" in val -%}
|
| 30 |
+
{%- for line in val['description'].split('\n') -%}
|
| 31 |
+
{%- if line.strip() -%}
|
| 32 |
+
{{- indent + '// ' + line + '\n' -}}
|
| 33 |
+
{%- endif -%}
|
| 34 |
+
{%- endfor -%}
|
| 35 |
+
{%- endif -%}
|
| 36 |
+
|
| 37 |
+
{# ---------- Additional JSON Keys ---------- #}
|
| 38 |
+
{%- for add_key, add_val in val.items() -%}
|
| 39 |
+
{%- if add_key in ADDITIONAL_JSON_KEYS -%}
|
| 40 |
+
{%- if add_val is string -%}
|
| 41 |
+
{{- indent + '// ' + add_key + ': "' + add_val + '"' + '\n' -}}
|
| 42 |
+
{%- else -%}
|
| 43 |
+
{{- indent + '// ' + add_key + ': ' ~ add_val ~ '\n' -}}
|
| 44 |
+
{%- endif -%}
|
| 45 |
+
{%- endif -%}
|
| 46 |
+
{%- endfor -%}
|
| 47 |
+
|
| 48 |
+
{# ---------- Property Definition ---------- #}
|
| 49 |
+
{%- set type_str = json_schema_to_typescript(
|
| 50 |
+
val,
|
| 51 |
+
indent + " "
|
| 52 |
+
) -%}
|
| 53 |
+
|
| 54 |
+
{{- indent + key + ('' if key in required else '?') + ': ' + type_str + ',' -}}
|
| 55 |
+
|
| 56 |
+
{%- if "default" in val or "defalut_value" in val -%}
|
| 57 |
+
{%- set default = val.get("default", val.get("defalut_value")) -%}
|
| 58 |
+
{%- if default is string -%}
|
| 59 |
+
{{- ' // default: "' + default + '"' -}}
|
| 60 |
+
{%- else -%}
|
| 61 |
+
{{- ' // default: ' ~ default -}}
|
| 62 |
+
{%- endif -%}
|
| 63 |
+
{%- endif -%}
|
| 64 |
+
|
| 65 |
+
{{- "\n" -}}
|
| 66 |
+
{%- endfor -%}
|
| 67 |
+
|
| 68 |
+
{# Handle additionalProperties as index signature #}
|
| 69 |
+
{%- if has_additional_props and additional_props_type is not none -%}
|
| 70 |
+
{%- set additional_type_str = json_schema_to_typescript(
|
| 71 |
+
additional_props_type,
|
| 72 |
+
indent + " "
|
| 73 |
+
) -%}
|
| 74 |
+
{{- indent + '[key: string]: ' + additional_type_str + '\n' -}}
|
| 75 |
+
{%- endif -%}
|
| 76 |
+
|
| 77 |
+
{{- indent[: (indent|length - " "|length) ] + '}' -}}
|
| 78 |
+
|
| 79 |
+
{# ---------------- STRING ---------------- #}
|
| 80 |
+
{%- elif ty == "string" -%}
|
| 81 |
+
{%- if schema.get("enum") -%}
|
| 82 |
+
{%- set ns = namespace(enum = []) -%}
|
| 83 |
+
{%- for en in schema['enum'] -%}
|
| 84 |
+
{%- set ns.enum = ns.enum + ['"' ~ en ~ '"'] -%}
|
| 85 |
+
{%- endfor -%}
|
| 86 |
+
{{- ns.enum | join(' | ') -}}
|
| 87 |
+
{%- elif schema.get("format", "none") in ['date-time', 'date'] -%}
|
| 88 |
+
{{- 'Date' -}}
|
| 89 |
+
{%- else -%}
|
| 90 |
+
{{- 'string' -}}
|
| 91 |
+
{%- endif -%}
|
| 92 |
+
|
| 93 |
+
{# ---------------- NUMBER / INTEGER ---------------- #}
|
| 94 |
+
{%- elif ty in ["number", "integer"] -%}
|
| 95 |
+
{%- if schema.get("enum") -%}
|
| 96 |
+
{{- schema.enum | join(' | ') -}}
|
| 97 |
+
{%- else -%}
|
| 98 |
+
{{- 'number' -}}
|
| 99 |
+
{%- endif -%}
|
| 100 |
+
|
| 101 |
+
{# ---------------- BOOLEAN ---------------- #}
|
| 102 |
+
{%- elif ty == "boolean" -%}
|
| 103 |
+
{{- 'boolean' -}}
|
| 104 |
+
|
| 105 |
+
{# ---------------- ARRAY ---------------- #}
|
| 106 |
+
{%- elif ty == "array" -%}
|
| 107 |
+
{%- if "items" in schema -%}
|
| 108 |
+
{{- json_schema_to_typescript(schema['items'], indent) + '[]' -}}
|
| 109 |
+
{%- else -%}
|
| 110 |
+
{{- 'Array<any>' -}}
|
| 111 |
+
{%- endif -%}
|
| 112 |
+
|
| 113 |
+
{# ---------------- FALLBACK ---------------- #}
|
| 114 |
+
{%- else -%}
|
| 115 |
+
{{- 'any' -}}
|
| 116 |
+
{%- endif -%}
|
| 117 |
+
{%- endmacro -%}
|
| 118 |
+
|
| 119 |
+
{#---------------------------------------------------------------
|
| 120 |
+
Renders a namespace and its tool definitions in TypeScript style
|
| 121 |
+
----------------------------------------------------------------#}
|
| 122 |
+
|
| 123 |
+
{%- macro render_tool_namespace(namespace_name, tools) -%}
|
| 124 |
+
{%- set ns = namespace(sections = ['namespace ' ~ namespace_name ~ ' {']) -%}
|
| 125 |
+
|
| 126 |
+
{%- for tool in tools -%}
|
| 127 |
+
{%- if tool.function -%}
|
| 128 |
+
{%- set tool = tool.function -%}
|
| 129 |
+
{%- endif -%}
|
| 130 |
+
|
| 131 |
+
{%- set ns_tool = namespace(content_lines=[]) -%}
|
| 132 |
+
|
| 133 |
+
{# ---------- TOOL DESCRIPTION ---------- #}
|
| 134 |
+
{%- if tool.get('description') -%}
|
| 135 |
+
{%- for line in tool['description'].split('\n') -%}
|
| 136 |
+
{%- if line.strip() -%}
|
| 137 |
+
{%- set ns_tool.content_lines = ns_tool.content_lines + ['// ' ~ line] -%}
|
| 138 |
+
{%- endif -%}
|
| 139 |
+
{%- endfor -%}
|
| 140 |
+
{%- endif -%}
|
| 141 |
+
|
| 142 |
+
{# ---------- TOOL SIGNATURE ---------- #}
|
| 143 |
+
{%- set main_body = "" -%}
|
| 144 |
+
{%- set params = tool.get("parameters") -%}
|
| 145 |
+
{%- if params and params.get("properties") -%}
|
| 146 |
+
{%- set param_type = json_schema_to_typescript(params, " ") -%}
|
| 147 |
+
{%- set main_body = 'type ' ~ tool.name ~ ' = (_: ' ~ param_type ~ ') => ' -%}
|
| 148 |
+
{%- else -%}
|
| 149 |
+
{%- set main_body = 'type ' ~ tool.name ~ ' = () => ' -%}
|
| 150 |
+
{%- endif -%}
|
| 151 |
+
|
| 152 |
+
{# ---------- RETURN TYPE ---------- #}
|
| 153 |
+
{%- set return_params = tool.get("return_parameters") -%}
|
| 154 |
+
{%- if return_params and return_params.get("properties") -%}
|
| 155 |
+
{%- set return_type = json_schema_to_typescript(return_params, " ") -%}
|
| 156 |
+
{%- set main_body = main_body ~ return_type -%}
|
| 157 |
+
{%- else -%}
|
| 158 |
+
{%- set main_body = main_body ~ 'any' -%}
|
| 159 |
+
{%- endif -%}
|
| 160 |
+
|
| 161 |
+
{%- set main_body = main_body ~ ';\n' -%}
|
| 162 |
+
|
| 163 |
+
{%- set ns_tool.content_lines = ns_tool.content_lines + [main_body] -%}
|
| 164 |
+
|
| 165 |
+
{# ---------- ADD TOOL TO SECTIONS ---------- #}
|
| 166 |
+
{%- set ns.sections = ns.sections + [ns_tool.content_lines | join('\n')] -%}
|
| 167 |
+
{%- endfor -%}
|
| 168 |
+
|
| 169 |
+
{%- set ns.sections = ns.sections + ['} // namespace ' ~ namespace_name] -%}
|
| 170 |
+
|
| 171 |
+
{{- ns.sections | join('\n') -}}
|
| 172 |
+
{%- endmacro -%}
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
{# ----------- MESSAGE RENDERING HELPER FUNCTIONS ------------ #}
|
| 176 |
+
|
| 177 |
+
{%- macro render_function_call(call) -%}
|
| 178 |
+
{%- if call.function -%}
|
| 179 |
+
{%- set call = call.function -%}
|
| 180 |
+
{%- endif -%}
|
| 181 |
+
|
| 182 |
+
{%- set arguments = call['arguments'] -%}
|
| 183 |
+
{%- if arguments is not string -%}
|
| 184 |
+
{%- set arguments = arguments| tojson(ensure_ascii=False) -%}
|
| 185 |
+
{%- endif -%}
|
| 186 |
+
|
| 187 |
+
{{- '{"name": "' ~ call['name'] ~ '", "arguments": ' ~ arguments ~ '}' -}}
|
| 188 |
+
{%- endmacro -%}
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
{%- macro render_role_message(message, role=None) -%}
|
| 192 |
+
{%- if not role -%}
|
| 193 |
+
{%- set role = message["role"] -%}
|
| 194 |
+
{%- endif -%}
|
| 195 |
+
|
| 196 |
+
{%- set message_content = message['content'] or '' -%}
|
| 197 |
+
{%- if message_content is not string -%}
|
| 198 |
+
{%- set message_content = message_content | tojson(ensure_ascii=False) -%}
|
| 199 |
+
{%- endif -%}
|
| 200 |
+
|
| 201 |
+
{{- role + add_tokens.role_sep + message_content -}}
|
| 202 |
+
|
| 203 |
+
{%- if message.tool_calls is defined and message.tool_calls -%}
|
| 204 |
+
{{- add_tokens.function_call + render_function_call(message.tool_calls[0]) -}}
|
| 205 |
+
{%- endif -%}
|
| 206 |
+
|
| 207 |
+
{{- add_tokens.message_sep -}}
|
| 208 |
+
|
| 209 |
+
{%- endmacro -%}
|
| 210 |
+
|
| 211 |
+
|
| 212 |
+
|
| 213 |
+
{# ----- SPECIAL TOKENS ----- #}
|
| 214 |
+
|
| 215 |
+
{%- set add_tokens = namespace(
|
| 216 |
+
role_sep="<|role_sep|>\n",
|
| 217 |
+
message_sep="<|message_sep|>\n\n",
|
| 218 |
+
function_call="<|function_call|>"
|
| 219 |
+
) -%}
|
| 220 |
+
|
| 221 |
+
{# ----- DEFAULT DEVSYSTEM ----- #}
|
| 222 |
+
|
| 223 |
+
{%- set DEVSYSTEM -%}
|
| 224 |
+
<role_description>
|
| 225 |
+
Description of the roles available in the dialog.
|
| 226 |
+
|
| 227 |
+
`developer system`
|
| 228 |
+
A message added by Sber before the main dialog. It has the highest priority and sets global, non-overridable conditions (for example, conversation rules, the safety policy, the assistant's overall response style, etc.).
|
| 229 |
+
|
| 230 |
+
`system`
|
| 231 |
+
A system instruction added by developers or by the user, but with a lower priority than `developer system`. It usually describes the assistant's instructions, a specific response style, and other conditions for this particular dialog.
|
| 232 |
+
|
| 233 |
+
`user`
|
| 234 |
+
A message or request from the user. The assistant follows it if it does not conflict with higher-priority instructions (see <instruction_priority>).
|
| 235 |
+
|
| 236 |
+
`user memory`
|
| 237 |
+
A sequence of the most up-to-date long-term facts about the user at the time of their request, presented as a JSON list of strings. Facts are listed in chronological order, meaning newer facts are appended to the end of the sequence. When facts are changed or deleted, records of previous facts remain in the sequence. The assistant saves facts using a function and uses them in accordance with the <memory_guidelines> block below.
|
| 238 |
+
|
| 239 |
+
`added files`
|
| 240 |
+
Metadata about files available for use in the dialog, presented in JSON format. It contains the following keys: id (a unique file identifier), name (file name), type (file type).
|
| 241 |
+
|
| 242 |
+
`assistant`
|
| 243 |
+
The assistant's reply to the user's request. If the system instruction or the user does not set additional rules for `assistant`, this reply must comply with the instructions in the <assistant_guidelines> block below. The list of functions available to call is contained in `function descriptions`. The name of the required function and its arguments will be generated next by the `function call` role. In its replies, the assistant follows the instructions in accordance with <instruction_priority>.
|
| 244 |
+
|
| 245 |
+
`function descriptions`
|
| 246 |
+
Function descriptions in TypeScript format. A function is a special tool (or a set of instructions) that the assistant can call to perform specific actions, computations, or obtain data needed to solve the user's task. Each function description contains blocks with the name, description, and arguments. Sometimes the description contains separate blocks with return parameters and usage examples that illustrate the correct call and arguments.
|
| 247 |
+
|
| 248 |
+
`function call`
|
| 249 |
+
The function that `assistant` calls based on the dialog context, and its arguments. The function is invoked in strict accordance with the instructions in the <function_usage> block.
|
| 250 |
+
|
| 251 |
+
`function result`
|
| 252 |
+
The result of the last function call.
|
| 253 |
+
</role_description>
|
| 254 |
+
|
| 255 |
+
<available_modalities>
|
| 256 |
+
The assistant can work with the following modalities: text, available functions.
|
| 257 |
+
</available_modalities>
|
| 258 |
+
|
| 259 |
+
<instruction_priority>
|
| 260 |
+
If instructions from different roles conflict within the dialog context, observe the following priorities:
|
| 261 |
+
`developer system` > `system` > `user` > `function descriptions` > `function result` > `user memory`
|
| 262 |
+
</instruction_priority>
|
| 263 |
+
|
| 264 |
+
<function_usage>
|
| 265 |
+
Basic instructions for working with functions.
|
| 266 |
+
|
| 267 |
+
Only call those functions that are described in `function descriptions`.
|
| 268 |
+
|
| 269 |
+
Call available functions when, according to their description, such a call will help provide a more complete and/or accurate answer to the user's request. Fill in function arguments using information from the dialog context. If a function could help answer the request but a required argument is missing from the context, ask the user for the missing data before calling the function. If a necessary function is unavailable or an error occurs, briefly inform the user and, if possible, suggest an alternative.
|
| 270 |
+
</function_usage>
|
| 271 |
+
|
| 272 |
+
<memory_guidelines>
|
| 273 |
+
Rules for using facts in long-term memory:
|
| 274 |
+
|
| 275 |
+
If there is no message under the `user memory` role in the dialog, this is equivalent to the absence of long-term facts about the user in memory. In that case, information about the user is limited to the current dialog, and no new facts should be saved.
|
| 276 |
+
</memory_guidelines>
|
| 277 |
+
|
| 278 |
+
<assistant_guidelines>
|
| 279 |
+
You are a helpful assistant.
|
| 280 |
+
|
| 281 |
+
# Instructions
|
| 282 |
+
- Strictly follow the instruction priority.
|
| 283 |
+
- Maintain a logical chain of reasoning when answering the user's question.
|
| 284 |
+
- For complex questions (for example, STEM), try to answer in detail unless the system message or dialog context limits the response length.
|
| 285 |
+
- Be helpful, truthful, and avoid unsafe or prohibited content in your responses.
|
| 286 |
+
- Try to reply in the language in which the user asked their question.
|
| 287 |
+
</assistant_guidelines>
|
| 288 |
+
|
| 289 |
+
A dialog will follow below.
|
| 290 |
+
The dialog may include various roles described in the <role_description> block.
|
| 291 |
+
Each turn begins with the role name and a special token that marks the end of the role's full name, and ends with a special end-of-turn token.
|
| 292 |
+
Your task is to continue the dialog from the last specified role in accordance with the dialog context.
|
| 293 |
+
{%- endset -%}
|
| 294 |
+
|
| 295 |
+
|
| 296 |
+
{#- ---------------------- RENDERING STARTS HERE ---------------------- -#}
|
| 297 |
+
|
| 298 |
+
|
| 299 |
+
{# ----- RENDER BOS TOKEN ----- #}
|
| 300 |
+
{{- bos_token -}}
|
| 301 |
+
|
| 302 |
+
|
| 303 |
+
{# ----- RENDER DEVSYSTEM ----- #}
|
| 304 |
+
{{- render_role_message({"role": "developer system", "content": DEVSYSTEM}) -}}
|
| 305 |
+
|
| 306 |
+
{# ----- RENDER SYSTEM IF PRESENT ----- #}
|
| 307 |
+
{%- if messages and messages[0]['role'] == 'system' -%}
|
| 308 |
+
{{- render_role_message(messages[0]) -}}
|
| 309 |
+
{%- set messages = messages[1:] -%}
|
| 310 |
+
{%- endif -%}
|
| 311 |
+
|
| 312 |
+
{# ----- RENDER TOOLS ----- #}
|
| 313 |
+
{%- if tools -%}
|
| 314 |
+
{%- set tools_content = (
|
| 315 |
+
render_tool_namespace('functions', tools)
|
| 316 |
+
+ "\n\n"
|
| 317 |
+
) -%}
|
| 318 |
+
{{- render_role_message({'role': 'function descriptions', 'content': tools_content}) -}}
|
| 319 |
+
{%- endif -%}
|
| 320 |
+
|
| 321 |
+
{# ----- MAIN MESSAGE LOOP ----- #}
|
| 322 |
+
{%- for message in messages -%}
|
| 323 |
+
|
| 324 |
+
{# ----- TOOL MESSAGE -------#}
|
| 325 |
+
{%- if message['role'] == 'tool' -%}
|
| 326 |
+
{{- render_role_message(message, 'function result') -}}
|
| 327 |
+
|
| 328 |
+
{# ----- OTHER MESSAGES ----- #}
|
| 329 |
+
{%- else -%}
|
| 330 |
+
{{- render_role_message(message) -}}
|
| 331 |
+
{%- endif -%}
|
| 332 |
+
|
| 333 |
+
{# ----- ADDING GENERATION PROMPT ----- #}
|
| 334 |
+
|
| 335 |
+
{%- if loop.last and add_generation_prompt and message['role'] != 'assistant' -%}
|
| 336 |
+
{{- 'assistant' + add_tokens.role_sep -}}
|
| 337 |
+
{%- endif -%}
|
| 338 |
+
|
| 339 |
+
{%- endfor -%}
|
models/templates/HuggingFaceTB-SmolLM3-3B.jinja
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{#- Copyright 2025-present the Unsloth team. All rights reserved. #}
|
| 2 |
+
{#- Licensed under the Apache License, Version 2.0 (the "License") #}
|
| 3 |
+
{#- Edits made by Unsloth to make it work for most inference engines #}
|
| 4 |
+
{# ───── defaults ───── #}
|
| 5 |
+
{%- if enable_thinking is not defined -%}
|
| 6 |
+
{%- set enable_thinking = true -%}
|
| 7 |
+
{%- endif -%}
|
| 8 |
+
{# ───── reasoning mode ───── #}
|
| 9 |
+
{%- if enable_thinking -%}
|
| 10 |
+
{%- set reasoning_mode = "/think" -%}
|
| 11 |
+
{%- else -%}
|
| 12 |
+
{%- set reasoning_mode = "/no_think" -%}
|
| 13 |
+
{%- endif -%}
|
| 14 |
+
{# ───── header (system message) ───── #}
|
| 15 |
+
{{- "<|im_start|>system\n" -}}
|
| 16 |
+
{%- if messages[0].role == "system" -%}
|
| 17 |
+
{%- set system_message = messages[0].content -%}
|
| 18 |
+
{%- if "/no_think" in system_message -%}
|
| 19 |
+
{%- set reasoning_mode = "/no_think" -%}
|
| 20 |
+
{%- elif "/think" in system_message -%}
|
| 21 |
+
{%- set reasoning_mode = "/think" -%}
|
| 22 |
+
{%- endif -%}
|
| 23 |
+
{%- set custom_instructions = system_message.replace("/no_think", "") -%}
|
| 24 |
+
{%- set custom_instructions = custom_instructions.replace("/think", "") -%}
|
| 25 |
+
{%- set custom_instructions = custom_instructions.rstrip() -%}
|
| 26 |
+
{%- endif -%}
|
| 27 |
+
{{- "## Metadata\n\n" -}}
|
| 28 |
+
{{- "Knowledge Cutoff Date: June 2025\n" -}}
|
| 29 |
+
{{- "Reasoning Mode: " + reasoning_mode + "\n\n" -}}
|
| 30 |
+
{{- "## Custom Instructions\n\n" -}}
|
| 31 |
+
{%- if custom_instructions -%}
|
| 32 |
+
{{- custom_instructions + "\n\n" -}}
|
| 33 |
+
{%- elif reasoning_mode == "/think" -%}
|
| 34 |
+
{{- "You are a helpful AI assistant named SmolLM, trained by Hugging Face. Your role as an assistant involves thoroughly exploring questions through a systematic thinking process before providing the final precise and accurate solutions. This requires engaging in a comprehensive cycle of analysis, summarizing, exploration, reassessment, reflection, backtracking, and iteration to develop well-considered thinking process. Please structure your response into two main sections: Thought and Solution using the specified format: <think> Thought section </think> Solution section. In the Thought section, detail your reasoning process in steps. Each step should include detailed considerations such as analysing questions, summarizing relevant findings, brainstorming new ideas, verifying the accuracy of the current steps, refining any errors, and revisiting previous steps. In the Solution section, based on various attempts, explorations, and reflections from the Thought section, systematically present the final solution that you deem correct. The Solution section should be logical, accurate, and concise and detail necessary steps needed to reach the conclusion.\n\n" -}}
|
| 35 |
+
{%- else -%}
|
| 36 |
+
{{- "You are a helpful AI assistant named SmolLM, trained by Hugging Face.\n\n" -}}
|
| 37 |
+
{%- endif -%}
|
| 38 |
+
{{- "<|im_end|>\n" -}}
|
| 39 |
+
{# ───── main loop ───── #}
|
| 40 |
+
{%- for message in messages -%}
|
| 41 |
+
{%- set content = message.content if message.content is string else "" -%}
|
| 42 |
+
{%- if message.role == "user" -%}
|
| 43 |
+
{{ "<|im_start|>" + message.role + "\n" + content + "<|im_end|>\n" }}
|
| 44 |
+
{%- elif message.role == "assistant" -%}
|
| 45 |
+
{%- if reasoning_mode == "/think" -%}
|
| 46 |
+
{{ "<|im_start|>assistant\n" + content.lstrip("\n") + "<|im_end|>\n" }}
|
| 47 |
+
{%- else -%}
|
| 48 |
+
{{ "<|im_start|>assistant\n" + "<think>\n\n</think>\n" + content.lstrip("\n") + "<|im_end|>\n" }}
|
| 49 |
+
{%- endif -%}
|
| 50 |
+
{%- elif message.role == "tool" -%}
|
| 51 |
+
{{ "<|im_start|>" + "user\n" + content + "<|im_end|>\n" }}
|
| 52 |
+
{%- endif -%}
|
| 53 |
+
{%- endfor -%}
|
| 54 |
+
{# ───── generation prompt ───── #}
|
| 55 |
+
{%- if add_generation_prompt -%}
|
| 56 |
+
{%- if reasoning_mode == "/think" -%}
|
| 57 |
+
{{ "<|im_start|>assistant\n" }}
|
| 58 |
+
{%- else -%}
|
| 59 |
+
{{ "<|im_start|>assistant\n" + "<think>\n\n</think>\n" }}
|
| 60 |
+
{%- endif -%}
|
| 61 |
+
{%- endif -%}
|
models/templates/Kimi-K2-Instruct.jinja
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{% macro render_content(msg) -%}
|
| 2 |
+
{%- set c = msg.get('content') -%}
|
| 3 |
+
{%- if c is string -%}
|
| 4 |
+
{{ c }}
|
| 5 |
+
{%- elif c is not none -%}
|
| 6 |
+
{% for content in c -%}
|
| 7 |
+
{% if content['type'] == 'image' or 'image' in content or 'image_url' in content -%}
|
| 8 |
+
<|media_start|>image<|media_content|><|media_pad|><|media_end|>
|
| 9 |
+
{% else -%}
|
| 10 |
+
{{ content['text'] }}
|
| 11 |
+
{%- endif -%}
|
| 12 |
+
{%- endfor -%}
|
| 13 |
+
{%- endif -%}
|
| 14 |
+
{%- endmacro %}
|
| 15 |
+
|
| 16 |
+
{%- set tool_response_queue = namespace(ids=[]) -%}
|
| 17 |
+
{%- set tool_call_counter = namespace(value=0) -%}
|
| 18 |
+
|
| 19 |
+
{%- if tools -%}
|
| 20 |
+
<|im_system|>tool_declare<|im_middle|>{{ tools | tojson }}<|im_end|>
|
| 21 |
+
{%- endif -%}
|
| 22 |
+
{% for message in messages %}
|
| 23 |
+
{%- if loop.first and messages[0]['role'] != 'system' -%}
|
| 24 |
+
<|im_system|>system<|im_middle|>You are Kimi, an AI assistant created by Moonshot AI.<|im_end|>
|
| 25 |
+
{% endif %}
|
| 26 |
+
|
| 27 |
+
{%- set role_name = message.get('name') or message['role'] -%}
|
| 28 |
+
{%- if message['role'] == 'user' -%}
|
| 29 |
+
<|im_user|>{{role_name}}<|im_middle|>
|
| 30 |
+
{%- elif message['role'] == 'assistant' -%}
|
| 31 |
+
<|im_assistant|>{{role_name}}<|im_middle|>
|
| 32 |
+
{%- else -%}
|
| 33 |
+
<|im_system|>{{role_name}}<|im_middle|>
|
| 34 |
+
{%- endif -%}
|
| 35 |
+
|
| 36 |
+
{%- if message['role'] == 'assistant' and message.get('tool_calls') -%}
|
| 37 |
+
{{render_content(message)}}<|tool_calls_section_begin|>
|
| 38 |
+
{%- for tool_call in message['tool_calls'] -%}
|
| 39 |
+
{%- set formatted_id = 'functions.' + tool_call['function']['name'] + ':' + (tool_call_counter.value | string) -%}
|
| 40 |
+
{%- set tool_call_counter.value = tool_call_counter.value + 1 -%}
|
| 41 |
+
{%- set _ = tool_response_queue.ids.append(formatted_id) -%}
|
| 42 |
+
<|tool_call_begin|>{{ formatted_id }}<|tool_call_argument_begin|>{% if tool_call['function']['arguments'] is string %}{{ tool_call['function']['arguments'] }}{% else %}{{ tool_call['function']['arguments'] | tojson }}{% endif %}<|tool_call_end|>
|
| 43 |
+
{%- endfor -%}
|
| 44 |
+
<|tool_calls_section_end|>
|
| 45 |
+
{%- elif message['role'] == 'tool' -%}
|
| 46 |
+
{%- if tool_response_queue.ids -%}
|
| 47 |
+
{%- set tool_call_id = tool_response_queue.ids.pop(0) -%}
|
| 48 |
+
{%- else -%}
|
| 49 |
+
{%- set tool_call_id = 'functions.' + message.get('name', 'unknown') + ':' + (tool_call_counter.value | string) -%}
|
| 50 |
+
{%- endif -%}
|
| 51 |
+
## Return of {{ tool_call_id }}
|
| 52 |
+
{{render_content(message)}}
|
| 53 |
+
{%- elif message['content'] is not none -%}
|
| 54 |
+
{{render_content(message)}}
|
| 55 |
+
{%- endif -%}
|
| 56 |
+
<|im_end|>
|
| 57 |
+
{%- endfor -%}
|
| 58 |
+
{%- if add_generation_prompt -%}
|
| 59 |
+
<|im_assistant|>assistant<|im_middle|>
|
| 60 |
+
{%- endif -%}
|
models/templates/Kimi-K2-Thinking.jinja
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{%- macro render_content(msg) -%}
|
| 2 |
+
{%- set c = msg.get('content') -%}
|
| 3 |
+
{%- if c is string -%}
|
| 4 |
+
{{ c }}
|
| 5 |
+
{%- elif c is not none -%}
|
| 6 |
+
{% for content in c -%}
|
| 7 |
+
{% if content['type'] == 'image' or 'image' in content or 'image_url' in content -%}
|
| 8 |
+
<|media_start|>image<|media_content|><|media_pad|><|media_end|>
|
| 9 |
+
{% else -%}
|
| 10 |
+
{{ content['text'] }}
|
| 11 |
+
{%- endif -%}
|
| 12 |
+
{%- endfor -%}
|
| 13 |
+
{%- endif -%}
|
| 14 |
+
{%- endmacro -%}
|
| 15 |
+
|
| 16 |
+
{% macro set_roles(message) -%}
|
| 17 |
+
{%- set role_name = message.get('name') or message['role'] -%}
|
| 18 |
+
{%- if message['role'] == 'user' -%}
|
| 19 |
+
<|im_user|>{{role_name}}<|im_middle|>
|
| 20 |
+
{%- elif message['role'] == 'assistant' -%}
|
| 21 |
+
<|im_assistant|>{{role_name}}<|im_middle|>
|
| 22 |
+
{%- else -%}
|
| 23 |
+
<|im_system|>{{role_name}}<|im_middle|>
|
| 24 |
+
{%- endif -%}
|
| 25 |
+
{%- endmacro -%}
|
| 26 |
+
|
| 27 |
+
{%- set tool_response_queue = namespace(ids=[]) -%}
|
| 28 |
+
{%- set tool_call_counter = namespace(value=0) -%}
|
| 29 |
+
|
| 30 |
+
{%- macro render_toolcalls(message) -%}
|
| 31 |
+
<|tool_calls_section_begin|>
|
| 32 |
+
{%- for tool_call in message['tool_calls'] -%}
|
| 33 |
+
{%- set formatted_id = 'functions.' + tool_call['function']['name'] + ':' + (tool_call_counter.value | string) -%}
|
| 34 |
+
{%- set tool_call_counter.value = tool_call_counter.value + 1 -%}
|
| 35 |
+
{%- set _ = tool_response_queue.ids.append(formatted_id) -%}
|
| 36 |
+
<|tool_call_begin|>{{ formatted_id }}<|tool_call_argument_begin|>{% if tool_call['function']['arguments'] is string %}{{ tool_call['function']['arguments'] }}{% else %}{{ tool_call['function']['arguments'] | tojson }}{% endif %}<|tool_call_end|>
|
| 37 |
+
{%- endfor -%}
|
| 38 |
+
<|tool_calls_section_end|>
|
| 39 |
+
{%- endmacro -%}
|
| 40 |
+
|
| 41 |
+
|
| 42 |
+
{# Find last non-tool-call assisitant message #}
|
| 43 |
+
{%- set ns = namespace(last_non_tool_call_assistant_msg=-1) -%}
|
| 44 |
+
{%- for idx in range(messages|length-1, -1, -1) -%}
|
| 45 |
+
{%- if messages[idx]['role'] == 'assistant' and not messages[idx].get('tool_calls') -%}
|
| 46 |
+
{%- set ns.last_non_tool_call_assistant_msg = idx -%}
|
| 47 |
+
{%- endif -%}
|
| 48 |
+
{%- endfor -%}
|
| 49 |
+
|
| 50 |
+
{# split all messages into history & suffix, reasoning_content in suffix should be reserved.#}
|
| 51 |
+
{%- set hist_msgs = messages[:ns.last_non_tool_call_assistant_msg+1] -%}
|
| 52 |
+
{%- set suffix_msgs = messages[ns.last_non_tool_call_assistant_msg+1:] -%}
|
| 53 |
+
|
| 54 |
+
{%- if tools -%}
|
| 55 |
+
<|im_system|>tool_declare<|im_middle|>{{ tools | tojson }}<|im_end|>
|
| 56 |
+
{%- endif -%}
|
| 57 |
+
|
| 58 |
+
{%- if messages|length == 0 or messages[0]['role'] != 'system' -%}
|
| 59 |
+
<|im_system|>system<|im_middle|>You are Kimi, an AI assistant created by Moonshot AI.<|im_end|>
|
| 60 |
+
{%- endif -%}
|
| 61 |
+
|
| 62 |
+
{%- for message in hist_msgs -%}
|
| 63 |
+
{{set_roles(message)}}
|
| 64 |
+
{%- if message['role'] == 'assistant' -%}
|
| 65 |
+
<think></think>{{render_content(message)}}
|
| 66 |
+
{%- if message.get('tool_calls') -%}
|
| 67 |
+
{{render_toolcalls(message)}}
|
| 68 |
+
{%- endif -%}
|
| 69 |
+
{%- elif message['role'] == 'tool' -%}
|
| 70 |
+
{%- if tool_response_queue.ids -%}
|
| 71 |
+
{%- set tool_call_id = tool_response_queue.ids.pop(0) -%}
|
| 72 |
+
{%- else -%}
|
| 73 |
+
{%- set tool_call_id = 'functions.' + message.get('name', 'unknown') + ':' + (tool_call_counter.value | string) -%}
|
| 74 |
+
{%- endif -%}
|
| 75 |
+
## Return of {{ tool_call_id }}
|
| 76 |
+
{{render_content(message)}}
|
| 77 |
+
{%- elif message['content'] is not none -%}
|
| 78 |
+
{{render_content(message)}}
|
| 79 |
+
{%- endif -%}
|
| 80 |
+
<|im_end|>
|
| 81 |
+
{%- endfor -%}
|
| 82 |
+
|
| 83 |
+
{%- for message in suffix_msgs -%}
|
| 84 |
+
{{set_roles(message)}}
|
| 85 |
+
{%- if message['role'] == 'assistant' -%}
|
| 86 |
+
{%- set rc = message.get('reasoning_content', '') -%}
|
| 87 |
+
<think>{{rc}}</think>{{render_content(message)}}
|
| 88 |
+
{%- if message.get('tool_calls') -%}
|
| 89 |
+
{{render_toolcalls(message)}}
|
| 90 |
+
{%- endif -%}
|
| 91 |
+
{%- elif message['role'] == 'tool' -%}
|
| 92 |
+
{%- if tool_response_queue.ids -%}
|
| 93 |
+
{%- set tool_call_id = tool_response_queue.ids.pop(0) -%}
|
| 94 |
+
{%- else -%}
|
| 95 |
+
{%- set tool_call_id = 'functions.' + message.get('name', 'unknown') + ':' + (tool_call_counter.value | string) -%}
|
| 96 |
+
{%- endif -%}
|
| 97 |
+
## Return of {{ tool_call_id }}
|
| 98 |
+
{{render_content(message)}}
|
| 99 |
+
{%- elif message['content'] is not none -%}
|
| 100 |
+
{{render_content(message)}}
|
| 101 |
+
{%- endif -%}
|
| 102 |
+
<|im_end|>
|
| 103 |
+
{%- endfor -%}
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
{%- if add_generation_prompt -%}
|
| 107 |
+
<|im_assistant|>assistant<|im_middle|>
|
| 108 |
+
{%- endif -%}
|
models/templates/Kimi-K3.jinja
ADDED
|
@@ -0,0 +1,324 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
{%- macro escape_attr(value) -%}
|
| 2 |
+
{{- value|string|replace('&', '&')|replace('"', '"') -}}
|
| 3 |
+
{%- endmacro -%}
|
| 4 |
+
|
| 5 |
+
{%- macro open_tag(tag, attrs=[]) -%}
|
| 6 |
+
{{- '<|open|>' + tag -}}
|
| 7 |
+
{%- for attr in attrs -%}
|
| 8 |
+
{{- ' ' + attr[0] + '="' -}}{{- escape_attr(attr[1]) -}}{{- '"' -}}
|
| 9 |
+
{%- endfor -%}
|
| 10 |
+
{{- '<|sep|>' -}}
|
| 11 |
+
{%- endmacro -%}
|
| 12 |
+
|
| 13 |
+
{%- macro close_tag(tag) -%}
|
| 14 |
+
{{- '<|close|>' + tag + '<|sep|>' -}}
|
| 15 |
+
{%- endmacro -%}
|
| 16 |
+
|
| 17 |
+
{%- macro next_image(state) -%}
|
| 18 |
+
{%- if image_prompts is defined and image_prompts is not none -%}
|
| 19 |
+
{%- if state.image_index >= image_prompts|length -%}
|
| 20 |
+
{{- raise_exception('More image placeholders than image prompts.') -}}
|
| 21 |
+
{%- endif -%}
|
| 22 |
+
{{- image_prompts[state.image_index] -}}
|
| 23 |
+
{%- set state.image_index = state.image_index + 1 -%}
|
| 24 |
+
{%- else -%}
|
| 25 |
+
{{- '<|kimi_image_placeholder|>' -}}
|
| 26 |
+
{%- endif -%}
|
| 27 |
+
{%- endmacro -%}
|
| 28 |
+
|
| 29 |
+
{%- macro render_text(text, state) -%}
|
| 30 |
+
{%- set text = text|string -%}
|
| 31 |
+
{%- if image_prompts is defined and image_prompts is not none and '<|kimi_image_placeholder|>' in text -%}
|
| 32 |
+
{%- set parts = text.split('<|kimi_image_placeholder|>') -%}
|
| 33 |
+
{%- for part in parts -%}
|
| 34 |
+
{{- part -}}
|
| 35 |
+
{%- if not loop.last -%}{{- next_image(state) -}}{%- endif -%}
|
| 36 |
+
{%- endfor -%}
|
| 37 |
+
{%- else -%}
|
| 38 |
+
{{- text -}}
|
| 39 |
+
{%- endif -%}
|
| 40 |
+
{%- endmacro -%}
|
| 41 |
+
|
| 42 |
+
{%- macro render_content(content, state) -%}
|
| 43 |
+
{%- if content is string -%}
|
| 44 |
+
{{- render_text(content, state) -}}
|
| 45 |
+
{%- elif content is not none and content is defined -%}
|
| 46 |
+
{%- for part in content -%}
|
| 47 |
+
{%- if part.type in ['image', 'image_url'] -%}
|
| 48 |
+
{{- next_image(state) -}}
|
| 49 |
+
{%- else -%}
|
| 50 |
+
{{- render_text(part.text, state) -}}
|
| 51 |
+
{%- endif -%}
|
| 52 |
+
{%- endfor -%}
|
| 53 |
+
{%- endif -%}
|
| 54 |
+
{%- endmacro -%}
|
| 55 |
+
|
| 56 |
+
{%- macro internal_system_message(message_type, body) -%}
|
| 57 |
+
{{- open_tag('message', [('role', 'system'), ('type', message_type)]) -}}
|
| 58 |
+
{{- body|trim -}}
|
| 59 |
+
{{- close_tag('message') -}}
|
| 60 |
+
{{- '<|end_of_msg|>' -}}
|
| 61 |
+
{%- endmacro -%}
|
| 62 |
+
|
| 63 |
+
{%- macro json_sorted(value) -%}
|
| 64 |
+
{#- tojson has no sort_keys, so sort each mapping level with dictsort to match the
|
| 65 |
+
reference implementation. Array order is kept as-is. -#}
|
| 66 |
+
{%- if value is mapping -%}
|
| 67 |
+
{{- '{' -}}
|
| 68 |
+
{%- for key, item in value|dictsort -%}
|
| 69 |
+
{%- if not loop.first -%}{{- ',' -}}{%- endif -%}
|
| 70 |
+
{{- key|tojson(ensure_ascii=false) -}}{{- ':' -}}{{- json_sorted(item) -}}
|
| 71 |
+
{%- endfor -%}
|
| 72 |
+
{{- '}' -}}
|
| 73 |
+
{%- elif value is string or value is number or value is boolean or value is none -%}
|
| 74 |
+
{{- value|tojson(ensure_ascii=false) -}}
|
| 75 |
+
{%- else -%}
|
| 76 |
+
{{- '[' -}}
|
| 77 |
+
{%- for item in value -%}
|
| 78 |
+
{%- if not loop.first -%}{{- ',' -}}{%- endif -%}
|
| 79 |
+
{{- json_sorted(item) -}}
|
| 80 |
+
{%- endfor -%}
|
| 81 |
+
{{- ']' -}}
|
| 82 |
+
{%- endif -%}
|
| 83 |
+
{%- endmacro -%}
|
| 84 |
+
|
| 85 |
+
{%- macro render_tool_declare(tool_list, dynamic=false) -%}
|
| 86 |
+
{{- open_tag('message', [('role', 'system'), ('type', 'tool-declare')]) -}}
|
| 87 |
+
{%- if dynamic -%}
|
| 88 |
+
{{- '## New Tools Available\nThe system dynamically extends the toolset via lazy-loading.\nYou have access to all existing and extended tools.\nHere are the specs for the extended tools.\n\n```json\n' -}}
|
| 89 |
+
{%- else -%}
|
| 90 |
+
{{- '# Tools\nHere are the available tools, described in JSONSchema.\n\n```json\n' -}}
|
| 91 |
+
{%- endif -%}
|
| 92 |
+
{{- json_sorted(tool_list) -}}
|
| 93 |
+
{{- '\n```' -}}
|
| 94 |
+
{{- close_tag('message') -}}
|
| 95 |
+
{{- '<|end_of_msg|>' -}}
|
| 96 |
+
{%- endmacro -%}
|
| 97 |
+
|
| 98 |
+
{%- macro xtml_type(value) -%}
|
| 99 |
+
{%- if value is boolean -%}boolean
|
| 100 |
+
{%- elif value is none -%}null
|
| 101 |
+
{%- elif value is number -%}number
|
| 102 |
+
{%- elif value is string -%}string
|
| 103 |
+
{%- elif value is mapping -%}object
|
| 104 |
+
{%- else -%}array
|
| 105 |
+
{%- endif -%}
|
| 106 |
+
{%- endmacro -%}
|
| 107 |
+
|
| 108 |
+
{%- macro xtml_value(value) -%}
|
| 109 |
+
{%- if value is string -%}
|
| 110 |
+
{{- value -}}
|
| 111 |
+
{%- else -%}
|
| 112 |
+
{{- value|tojson(ensure_ascii=false) -}}
|
| 113 |
+
{%- endif -%}
|
| 114 |
+
{%- endmacro -%}
|
| 115 |
+
|
| 116 |
+
{%- macro render_assistant(message, state) -%}
|
| 117 |
+
{%- if thinking -%}
|
| 118 |
+
{%- set reasoning_content = message.get('reasoning_content') or message.get('reasoning') -%}
|
| 119 |
+
{{- open_tag('think') -}}
|
| 120 |
+
{%- if reasoning_content is not none and reasoning_content|string|trim -%}
|
| 121 |
+
{{- render_text(reasoning_content, state) -}}
|
| 122 |
+
{%- endif -%}
|
| 123 |
+
{{- close_tag('think') -}}
|
| 124 |
+
{%- endif -%}
|
| 125 |
+
{{- open_tag('response') -}}
|
| 126 |
+
{{- render_content(message.get('content'), state) -}}
|
| 127 |
+
{{- close_tag('response') -}}
|
| 128 |
+
{%- set tool_calls = message.get('tool_calls') -%}
|
| 129 |
+
{%- if tool_calls -%}
|
| 130 |
+
{{- open_tag('tools') -}}
|
| 131 |
+
{%- for tool_call in tool_calls -%}
|
| 132 |
+
{%- if tool_call is not mapping -%}
|
| 133 |
+
{{- raise_exception('Kimi K3 tool calls must be mappings.') -}}
|
| 134 |
+
{%- endif -%}
|
| 135 |
+
{%- set fn = tool_call.function if tool_call.function is defined and tool_call.function is mapping else tool_call -%}
|
| 136 |
+
{%- if fn.get('name') is none -%}
|
| 137 |
+
{{- raise_exception('Kimi K3 tool calls require a function name.') -}}
|
| 138 |
+
{%- endif -%}
|
| 139 |
+
{{- open_tag('call', [('tool', fn.name), ('index', loop.index)]) -}}
|
| 140 |
+
{%- set arguments = fn.get('arguments', {}) -%}
|
| 141 |
+
{%- set json_block = fn.get('_xtml_json_block') -%}
|
| 142 |
+
{%- if json_block is not none -%}
|
| 143 |
+
{{- open_tag('json', [('type', 'object')]) -}}
|
| 144 |
+
{{- render_text(json_block, state) -}}
|
| 145 |
+
{{- close_tag('json') -}}
|
| 146 |
+
{%- elif arguments is mapping -%}
|
| 147 |
+
{%- for key, value in arguments.items() -%}
|
| 148 |
+
{{- open_tag('argument', [('key', key), ('type', xtml_type(value))]) -}}
|
| 149 |
+
{{- render_text(xtml_value(value), state) -}}
|
| 150 |
+
{{- close_tag('argument') -}}
|
| 151 |
+
{%- endfor -%}
|
| 152 |
+
{%- elif arguments is string and arguments|trim -%}
|
| 153 |
+
{{- open_tag('json', [('type', 'object')]) -}}
|
| 154 |
+
{{- render_text(arguments, state) -}}
|
| 155 |
+
{{- close_tag('json') -}}
|
| 156 |
+
{%- elif arguments is not none and arguments is not string -%}
|
| 157 |
+
{{- raise_exception('Kimi K3 tool call arguments must be a mapping or a JSON object string.') -}}
|
| 158 |
+
{%- endif -%}
|
| 159 |
+
{{- close_tag('call') -}}
|
| 160 |
+
{%- endfor -%}
|
| 161 |
+
{{- close_tag('tools') -}}
|
| 162 |
+
{%- endif -%}
|
| 163 |
+
{%- endmacro -%}
|
| 164 |
+
|
| 165 |
+
{%- macro render_tool_message(message, state, resolved_name=none) -%}
|
| 166 |
+
{%- set state.tool_index = state.tool_index + 1 -%}
|
| 167 |
+
{%- if resolved_name is not none -%}
|
| 168 |
+
{%- set tool_name = resolved_name -%}
|
| 169 |
+
{%- elif 'tool' in message -%}
|
| 170 |
+
{%- set tool_name = message.get('tool') -%}
|
| 171 |
+
{%- else -%}
|
| 172 |
+
{%- set tool_name = message.get('name') -%}
|
| 173 |
+
{%- endif -%}
|
| 174 |
+
{%- if tool_name is none and state.tool_calls is not none and state.tool_index <= state.tool_calls|length -%}
|
| 175 |
+
{%- set fallback_call = state.tool_calls[state.tool_index - 1] -%}
|
| 176 |
+
{%- set fallback_fn = fallback_call.function if fallback_call.function is defined and fallback_call.function is mapping else fallback_call -%}
|
| 177 |
+
{%- set tool_name = fallback_fn.name -%}
|
| 178 |
+
{%- endif -%}
|
| 179 |
+
{%- if tool_name is none -%}
|
| 180 |
+
{{- raise_exception('Kimi K3 tool messages need a resolvable tool name: carry `tool`/`name`, or match a preceding assistant tool_call by order.') -}}
|
| 181 |
+
{%- endif -%}
|
| 182 |
+
{{- open_tag('message', [('role', 'tool'), ('tool', tool_name), ('index', state.tool_index)]) -}}
|
| 183 |
+
{{- render_content(message.get('content'), state) -}}
|
| 184 |
+
{{- close_tag('message') -}}
|
| 185 |
+
{{- '<|end_of_msg|>' -}}
|
| 186 |
+
{%- endmacro -%}
|
| 187 |
+
|
| 188 |
+
{%- if thinking is undefined -%}
|
| 189 |
+
{%- set thinking = true -%}
|
| 190 |
+
{%- endif -%}
|
| 191 |
+
{%- if thinking_effort is undefined -%}
|
| 192 |
+
{%- set thinking_effort = 'max' -%}
|
| 193 |
+
{%- endif -%}
|
| 194 |
+
{%- if thinking and thinking_effort is not none and thinking_effort not in ['low', 'high', 'max'] -%}
|
| 195 |
+
{{- raise_exception('Unsupported thinking_effort=' + thinking_effort|string + '; supported values are low, high, and max.') -}}
|
| 196 |
+
{%- endif -%}
|
| 197 |
+
|
| 198 |
+
{%- set state = namespace(image_index=0, tool_calls=none, tool_index=0, response_schema=none) -%}
|
| 199 |
+
|
| 200 |
+
{%- if tools is defined and tools -%}
|
| 201 |
+
{{- render_tool_declare(tools) -}}
|
| 202 |
+
{%- endif -%}
|
| 203 |
+
|
| 204 |
+
{%- if thinking and thinking_effort in ['low', 'high', 'max'] -%}
|
| 205 |
+
{{- internal_system_message(
|
| 206 |
+
'thinking-effort',
|
| 207 |
+
'`thinking_effort` guides on how much to think in your thinking channel (not including the response channel), supported values include `low`, `medium`, `high`, and `max`.\nNow the system is invoked with `thinking_effort=' + thinking_effort|string + '`.'
|
| 208 |
+
) -}}
|
| 209 |
+
{%- endif -%}
|
| 210 |
+
|
| 211 |
+
{%- for message in messages -%}
|
| 212 |
+
{%- if message is mapping -%}
|
| 213 |
+
{%- if 'role' not in message -%}
|
| 214 |
+
{{- raise_exception('Kimi K3 messages require a role.') -}}
|
| 215 |
+
{%- elif message.role == 'user' -%}
|
| 216 |
+
{%- set attrs = [('role', 'user')] -%}
|
| 217 |
+
{%- if message.get('name') -%}{%- set attrs = attrs + [('name', message.name)] -%}{%- endif -%}
|
| 218 |
+
{{- open_tag('message', attrs) -}}
|
| 219 |
+
{{- render_content(message.get('content'), state) -}}
|
| 220 |
+
{{- close_tag('message') -}}
|
| 221 |
+
{{- '<|end_of_msg|>' -}}
|
| 222 |
+
{%- elif message.role == 'system' and message.get('tools') -%}
|
| 223 |
+
{{- render_tool_declare(message.tools, dynamic=true) -}}
|
| 224 |
+
{%- elif message.role == 'system' -%}
|
| 225 |
+
{%- set attrs = [('role', 'system')] -%}
|
| 226 |
+
{%- if message.get('name') -%}{%- set attrs = attrs + [('name', message.name)] -%}{%- endif -%}
|
| 227 |
+
{{- open_tag('message', attrs) -}}
|
| 228 |
+
{{- render_content(message.get('content'), state) -}}
|
| 229 |
+
{{- close_tag('message') -}}
|
| 230 |
+
{{- '<|end_of_msg|>' -}}
|
| 231 |
+
{%- elif message.role == 'assistant' -%}
|
| 232 |
+
{%- set state.tool_calls = message.get('tool_calls') -%}
|
| 233 |
+
{%- set state.tool_index = 0 -%}
|
| 234 |
+
{%- set attrs = [('role', 'assistant')] -%}
|
| 235 |
+
{%- if message.get('name') -%}{%- set attrs = attrs + [('name', message.name)] -%}{%- endif -%}
|
| 236 |
+
{{- open_tag('message', attrs) -}}
|
| 237 |
+
{{- render_assistant(message, state) -}}
|
| 238 |
+
{{- close_tag('message') -}}
|
| 239 |
+
{{- '<|end_of_msg|>' -}}
|
| 240 |
+
{%- elif message.role == 'tool' and (loop.first or messages[loop.index0 - 1].role != 'tool') -%}
|
| 241 |
+
{%- set run = namespace(tool_messages=[], resolved_count=0) -%}
|
| 242 |
+
{%- for candidate in messages[loop.index0:] -%}
|
| 243 |
+
{%- if candidate is not mapping or candidate.role != 'tool' -%}{%- break -%}{%- endif -%}
|
| 244 |
+
{%- set run.tool_messages = run.tool_messages + [candidate] -%}
|
| 245 |
+
{%- set call_id = candidate.get('tool_call_id', candidate.get('id')) -%}
|
| 246 |
+
{%- set match = namespace(found=false) -%}
|
| 247 |
+
{%- if call_id is not none and state.tool_calls is not none -%}
|
| 248 |
+
{%- for tool_call in state.tool_calls -%}
|
| 249 |
+
{%- if not match.found and tool_call is mapping and tool_call.get('id') is not none and tool_call.get('id')|string == call_id|string -%}
|
| 250 |
+
{%- set match.found = true -%}
|
| 251 |
+
{%- endif -%}
|
| 252 |
+
{%- endfor -%}
|
| 253 |
+
{%- endif -%}
|
| 254 |
+
{%- if match.found -%}{%- set run.resolved_count = run.resolved_count + 1 -%}{%- endif -%}
|
| 255 |
+
{%- endfor -%}
|
| 256 |
+
{%- if run.tool_messages|length > 0 and run.resolved_count == run.tool_messages|length -%}
|
| 257 |
+
{%- set emitted = namespace(ids=[]) -%}
|
| 258 |
+
{%- for tool_call in state.tool_calls -%}
|
| 259 |
+
{%- if tool_call is mapping and tool_call.get('id') is not none and tool_call.get('id')|string not in emitted.ids -%}
|
| 260 |
+
{%- set emitted.ids = emitted.ids + [tool_call.get('id')|string] -%}
|
| 261 |
+
{%- set fn = tool_call.function if tool_call.function is defined and tool_call.function is mapping else tool_call -%}
|
| 262 |
+
{%- for tool_message in run.tool_messages -%}
|
| 263 |
+
{%- set result_id = tool_message.get('tool_call_id', tool_message.get('id')) -%}
|
| 264 |
+
{%- if result_id is not none and result_id|string == tool_call.get('id')|string -%}
|
| 265 |
+
{{- render_tool_message(tool_message, state, fn.get('name')) -}}
|
| 266 |
+
{%- endif -%}
|
| 267 |
+
{%- endfor -%}
|
| 268 |
+
{%- endif -%}
|
| 269 |
+
{%- endfor -%}
|
| 270 |
+
{%- else -%}
|
| 271 |
+
{%- for tool_message in run.tool_messages -%}
|
| 272 |
+
{{- render_tool_message(tool_message, state) -}}
|
| 273 |
+
{%- endfor -%}
|
| 274 |
+
{%- endif -%}
|
| 275 |
+
{%- endif -%}
|
| 276 |
+
{%- endif -%}
|
| 277 |
+
{%- endfor -%}
|
| 278 |
+
|
| 279 |
+
{%- if tool_choice is defined and tool_choice == 'required' -%}
|
| 280 |
+
{{- internal_system_message('tool-choice', 'The system is invoked with `tool_choice=required`.\nYou MUST call tools in the next message.') -}}
|
| 281 |
+
{%- elif tool_choice is defined and tool_choice == 'none' -%}
|
| 282 |
+
{{- internal_system_message('tool-choice', 'The system is invoked with `tool_choice=none`.\nYou MUST NOT call any tools in the next message.') -}}
|
| 283 |
+
{%- endif -%}
|
| 284 |
+
|
| 285 |
+
{%- if response_schema is defined -%}
|
| 286 |
+
{%- set state.response_schema = response_schema -%}
|
| 287 |
+
{%- elif response_format is defined and response_format is mapping and response_format.get('json_schema') is not none -%}
|
| 288 |
+
{%- set schema_wrapper = response_format.get('json_schema') -%}
|
| 289 |
+
{%- if schema_wrapper is mapping and 'schema' in schema_wrapper -%}
|
| 290 |
+
{%- set state.response_schema = schema_wrapper.get('schema') -%}
|
| 291 |
+
{%- elif schema_wrapper is mapping and 'json_schema' in schema_wrapper -%}
|
| 292 |
+
{%- set state.response_schema = schema_wrapper.get('json_schema') -%}
|
| 293 |
+
{%- else -%}
|
| 294 |
+
{%- set state.response_schema = schema_wrapper -%}
|
| 295 |
+
{%- endif -%}
|
| 296 |
+
{%- endif -%}
|
| 297 |
+
|
| 298 |
+
{%- set response_format_type = none -%}
|
| 299 |
+
{%- if response_format is defined and response_format is mapping -%}
|
| 300 |
+
{%- set response_format_type = response_format.get('type') -%}
|
| 301 |
+
{%- elif response_format is defined -%}
|
| 302 |
+
{%- set response_format_type = response_format -%}
|
| 303 |
+
{%- endif -%}
|
| 304 |
+
{%- if response_format_type == 'json_object' -%}
|
| 305 |
+
{{- internal_system_message(
|
| 306 |
+
'response-format',
|
| 307 |
+
'The system is invoked with `response_format=json_object`.\nYour response must be raw JSON data without markdown code blocks (```json) or any additional formatting.'
|
| 308 |
+
) -}}
|
| 309 |
+
{%- elif response_format_type == 'json_schema' -%}
|
| 310 |
+
{{- internal_system_message(
|
| 311 |
+
'response-format',
|
| 312 |
+
'The system is invoked with `response_format=json_schema`.\nYour response must be raw JSON data without markdown code blocks (```json) or any additional formatting.\nThe JSON data must match the following schema:\n```json\n' + json_sorted(state.response_schema) + '\n```'
|
| 313 |
+
) -}}
|
| 314 |
+
{%- endif -%}
|
| 315 |
+
|
| 316 |
+
{%- if add_generation_prompt -%}
|
| 317 |
+
{{- open_tag('message', [('role', 'assistant')]) -}}
|
| 318 |
+
{{- open_tag('think' if thinking else 'response') -}}
|
| 319 |
+
{%- endif -%}
|
| 320 |
+
|
| 321 |
+
{%- if image_prompts is defined and image_prompts is not none and state.image_index != image_prompts|length -%}
|
| 322 |
+
{{- raise_exception('image prompt count ' + image_prompts|length|string + ' != consumed placeholder count ' + state.image_index|string) -}}
|
| 323 |
+
{%- endif -%}
|
| 324 |
+
|